Coverage for sources/detextive/decoders.py: 100%
81 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-14 22:27 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-14 22:27 +0000
1# vim: set filetype=python fileencoding=utf-8:
2# -*- coding: utf-8 -*-
4#============================================================================#
5# #
6# Licensed under the Apache License, Version 2.0 (the "License"); #
7# you may not use this file except in compliance with the License. #
8# You may obtain a copy of the License at #
9# #
10# http://www.apache.org/licenses/LICENSE-2.0 #
11# #
12# Unless required by applicable law or agreed to in writing, software #
13# distributed under the License is distributed on an "AS IS" BASIS, #
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
15# See the License for the specific language governing permissions and #
16# limitations under the License. #
17# #
18#============================================================================#
21''' Conversion of bytes arrays to Unicode text. '''
24from . import __
25from . import charsets as _charsets
26from . import detectors as _detectors
27from . import exceptions as _exceptions
28from . import inference as _inference
29from . import lineseparators as _lineseparators
30from . import mimetypes as _mimetypes
31from . import nomina as _nomina
32from . import validation as _validation
34from .core import ( # isort: skip
35 BEHAVIORS_DEFAULT as _BEHAVIORS_DEFAULT,
36 BehaviorTristate as _BehaviorTristate,
37 BehaviorsArgument as _BehaviorsArgument,
38 CharsetResult as _CharsetResult,
39 CodecSpecifiers as _CodecSpecifiers,
40 MimetypeResult as _MimetypeResult,
41)
44_MIMETYPE_DEFAULT_TEXTUAL = 'text/plain'
47class DecodeInformResult( __.immut.DataclassObject ):
48 ''' Decoded text with supplemental inference metadata. '''
50 text: __.typx.Annotated[
51 str, __.ddoc.Doc( ''' Decoded text content. ''' )
52 ]
53 charset: __.typx.Annotated[
54 _CharsetResult, __.ddoc.Doc( ''' Charset used for decoding. ''' )
55 ]
56 mimetype: __.typx.Annotated[
57 _MimetypeResult, __.ddoc.Doc( ''' Inferred MIME type metadata. ''' )
58 ]
59 linesep: __.typx.Annotated[
60 __.typx.Optional[ _lineseparators.LineSeparators ],
61 __.ddoc.Doc( ''' Detected line separator from content sample. ''' ),
62 ]
65def decode( # noqa: PLR0913
66 content: _nomina.Content, /, *,
67 behaviors: _BehaviorsArgument = _BEHAVIORS_DEFAULT,
68 profile: _validation.ProfileArgument = _validation.PROFILE_TEXTUAL,
69 http_content_type: _nomina.HttpContentTypeArgument = __.absent,
70 location: _nomina.LocationArgument = __.absent,
71 charset_supplement: _nomina.CharsetSupplementArgument = __.absent,
72) -> str:
73 ''' Decodes bytes array to Unicode text.
75 Uses trial decoding and validation; does not provide default-return
76 semantics. The ``charset_supplement`` parameter is a trial hint and
77 not a fallback return value.
78 '''
79 _, httpct_charset = _parse_http_content_type( http_content_type )
80 return _decode_content_charset_result(
81 content, behaviors, profile,
82 httpct_charset = httpct_charset,
83 location = location,
84 charset_supplement = charset_supplement )[ 0 ]
87def decode_inform( # noqa: PLR0913
88 content: _nomina.Content, /, *,
89 behaviors: _BehaviorsArgument = _BEHAVIORS_DEFAULT,
90 profile: _validation.ProfileArgument = _validation.PROFILE_TEXTUAL,
91 mimetype_default: _nomina.MimetypeDefaultArgument = (
92 _MIMETYPE_DEFAULT_TEXTUAL ),
93 http_content_type: _nomina.HttpContentTypeArgument = __.absent,
94 location: _nomina.LocationArgument = __.absent,
95 charset_supplement: _nomina.CharsetSupplementArgument = __.absent,
96) -> DecodeInformResult:
97 ''' Decodes bytes and returns supplemental inference metadata. '''
98 httpct_mimetype, httpct_charset = (
99 _parse_http_content_type( http_content_type ) )
100 text, charset_result = _decode_content_charset_result(
101 content, behaviors, profile,
102 httpct_charset = httpct_charset,
103 location = location,
104 charset_supplement = charset_supplement )
105 mimetype_result = _infer_mimetype(
106 content, behaviors,
107 mimetype_default = mimetype_default,
108 httpct_mimetype = httpct_mimetype,
109 location = location,
110 charset = charset_result.charset )
111 linesep = _lineseparators.LineSeparators.detect_bytes( content )
112 return DecodeInformResult(
113 text = text,
114 charset = charset_result,
115 mimetype = mimetype_result,
116 linesep = linesep )
119def _attempt_decode_http_content_type(
120 content: _nomina.Content,
121 behaviors: _BehaviorsArgument,
122 profile: _validation.ProfileArgument, /, *,
123 httpct_charset: __.Absential[ __.typx.Optional[ str ] ],
124 location: _nomina.LocationArgument,
125) -> __.Absential[ tuple[ str, _CharsetResult ] ]:
126 error = _exceptions.ContentDecodeImpossibility( location = location )
127 if httpct_charset is None: raise error
128 if __.is_absent( httpct_charset ): return __.absent
129 behaviors_ = __.dcls.replace(
130 behaviors, trial_codecs = ( _CodecSpecifiers.FromInference, ) )
131 try:
132 text, result = _charsets.attempt_decodes(
133 content,
134 behaviors = behaviors_,
135 inference = httpct_charset,
136 location = location )
137 except _exceptions.ContentDecodeFailure: return __.absent
138 _validate_text(
139 text, result.confidence,
140 behaviors = behaviors, profile = profile, location = location )
141 return text, result
144def _decode_content_charset_result( # noqa: PLR0913
145 content: _nomina.Content,
146 behaviors: _BehaviorsArgument,
147 profile: _validation.ProfileArgument, /, *,
148 httpct_charset: __.Absential[ __.typx.Optional[ str ] ],
149 location: _nomina.LocationArgument,
150 charset_supplement: _nomina.CharsetSupplementArgument,
151) -> tuple[ str, _CharsetResult ]:
152 if content == b'':
153 return '', _CharsetResult( charset = 'utf-8', confidence = 1.0 )
154 charset: __.Absential[ str ] = __.absent
155 result: __.Absential[ _CharsetResult ] = __.absent
156 httpct_result: __.Absential[ tuple[ str, _CharsetResult ] ] = __.absent
157 httpct_result = _attempt_decode_http_content_type(
158 content, behaviors, profile,
159 httpct_charset = httpct_charset, location = location )
160 if not __.is_absent( httpct_result ): return httpct_result
161 behaviors_ = __.dcls.replace(
162 behaviors, trial_decode = _BehaviorTristate.Never )
163 if behaviors.charset_detect:
164 with __.ctxl.suppress( _exceptions.CharsetDetectFailure ):
165 result = _detectors.detect_charset_confidence(
166 content,
167 behaviors = behaviors_,
168 supplement = charset_supplement,
169 location = location )
170 if ( result.charset
171 and result.confidence >= behaviors.trial_decode_confidence
172 ): charset = result.charset
173 validator = __.funct.partial(
174 _validate_text_in_decode_attempt,
175 behaviors = behaviors,
176 profile = profile,
177 location = location )
178 return _charsets.attempt_decodes(
179 content,
180 behaviors = behaviors,
181 inference = charset,
182 supplement = charset_supplement,
183 location = location,
184 validator = validator )
187def _infer_mimetype( # noqa: PLR0913
188 content: _nomina.Content,
189 behaviors: _BehaviorsArgument, /, *,
190 mimetype_default: _nomina.MimetypeDefaultArgument,
191 httpct_mimetype: __.Absential[ str ],
192 location: _nomina.LocationArgument,
193 charset: __.typx.Optional[ str ],
194) -> _MimetypeResult:
195 charset_ = __.absent if charset is None else charset
196 if ( not __.is_absent( httpct_mimetype )
197 and _mimetypes.is_textual_mimetype( httpct_mimetype )
198 ):
199 return _MimetypeResult( mimetype = httpct_mimetype, confidence = 0.9 )
200 result: __.Absential[ _MimetypeResult ] = __.absent
201 if not __.is_absent( location ):
202 mimetype = _mimetypes.mimetype_from_location( location )
203 if ( not __.is_absent( mimetype )
204 and _mimetypes.is_textual_mimetype( mimetype )
205 ):
206 return _MimetypeResult( mimetype = mimetype, confidence = 0.9 )
207 if behaviors.mimetype_detect:
208 result = _detectors.detect_mimetype_confidence(
209 content,
210 behaviors = behaviors,
211 default = mimetype_default,
212 charset = charset_,
213 location = location )
214 if __.is_absent( result ):
215 return _MimetypeResult( mimetype = mimetype_default, confidence = 1.0 )
216 if _mimetypes.is_textual_mimetype( result.mimetype ): return result
217 return _MimetypeResult( mimetype = mimetype_default, confidence = 1.0 )
220def _parse_http_content_type(
221 http_content_type: _nomina.HttpContentTypeArgument
222) -> tuple[ __.Absential[ str ], __.Absential[ __.typx.Optional[ str ] ] ]:
223 if __.is_absent( http_content_type ):
224 return __.absent, __.absent
225 return _inference.parse_http_content_type( http_content_type )
228def _validate_text(
229 text: str, confidence: float, /, *,
230 behaviors: _BehaviorsArgument,
231 profile: _validation.ProfileArgument,
232 location: _nomina.LocationArgument,
233) -> None:
234 error = _exceptions.TextInvalidity( location = location )
235 should_validate = False
236 match behaviors.text_validate:
237 case _BehaviorTristate.Always:
238 should_validate = True
239 case _BehaviorTristate.AsNeeded:
240 should_validate = confidence < behaviors.text_validate_confidence
241 case _BehaviorTristate.Never: pass
242 if should_validate and not profile( text ): raise error
245def _validate_text_in_decode_attempt(
246 text: str, result: _CharsetResult, /, *,
247 behaviors: _BehaviorsArgument,
248 profile: _validation.ProfileArgument,
249 location: _nomina.LocationArgument,
250) -> None:
251 _validate_text(
252 text, 0.0,
253 behaviors = behaviors,
254 profile = profile,
255 location = location )