Coverage for sources/detextive/decoders.py: 100%

81 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-14 22:27 +0000

1# vim: set filetype=python fileencoding=utf-8: 

2# -*- coding: utf-8 -*- 

3 

4#============================================================================# 

5# # 

6# Licensed under the Apache License, Version 2.0 (the "License"); # 

7# you may not use this file except in compliance with the License. # 

8# You may obtain a copy of the License at # 

9# # 

10# http://www.apache.org/licenses/LICENSE-2.0 # 

11# # 

12# Unless required by applicable law or agreed to in writing, software # 

13# distributed under the License is distributed on an "AS IS" BASIS, # 

14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # 

15# See the License for the specific language governing permissions and # 

16# limitations under the License. # 

17# # 

18#============================================================================# 

19 

20 

21''' Conversion of bytes arrays to Unicode text. ''' 

22 

23 

24from . import __ 

25from . import charsets as _charsets 

26from . import detectors as _detectors 

27from . import exceptions as _exceptions 

28from . import inference as _inference 

29from . import lineseparators as _lineseparators 

30from . import mimetypes as _mimetypes 

31from . import nomina as _nomina 

32from . import validation as _validation 

33 

34from .core import ( # isort: skip 

35 BEHAVIORS_DEFAULT as _BEHAVIORS_DEFAULT, 

36 BehaviorTristate as _BehaviorTristate, 

37 BehaviorsArgument as _BehaviorsArgument, 

38 CharsetResult as _CharsetResult, 

39 CodecSpecifiers as _CodecSpecifiers, 

40 MimetypeResult as _MimetypeResult, 

41) 

42 

43 

44_MIMETYPE_DEFAULT_TEXTUAL = 'text/plain' 

45 

46 

47class DecodeInformResult( __.immut.DataclassObject ): 

48 ''' Decoded text with supplemental inference metadata. ''' 

49 

50 text: __.typx.Annotated[ 

51 str, __.ddoc.Doc( ''' Decoded text content. ''' ) 

52 ] 

53 charset: __.typx.Annotated[ 

54 _CharsetResult, __.ddoc.Doc( ''' Charset used for decoding. ''' ) 

55 ] 

56 mimetype: __.typx.Annotated[ 

57 _MimetypeResult, __.ddoc.Doc( ''' Inferred MIME type metadata. ''' ) 

58 ] 

59 linesep: __.typx.Annotated[ 

60 __.typx.Optional[ _lineseparators.LineSeparators ], 

61 __.ddoc.Doc( ''' Detected line separator from content sample. ''' ), 

62 ] 

63 

64 

65def decode( # noqa: PLR0913 

66 content: _nomina.Content, /, *, 

67 behaviors: _BehaviorsArgument = _BEHAVIORS_DEFAULT, 

68 profile: _validation.ProfileArgument = _validation.PROFILE_TEXTUAL, 

69 http_content_type: _nomina.HttpContentTypeArgument = __.absent, 

70 location: _nomina.LocationArgument = __.absent, 

71 charset_supplement: _nomina.CharsetSupplementArgument = __.absent, 

72) -> str: 

73 ''' Decodes bytes array to Unicode text. 

74 

75 Uses trial decoding and validation; does not provide default-return 

76 semantics. The ``charset_supplement`` parameter is a trial hint and 

77 not a fallback return value. 

78 ''' 

79 _, httpct_charset = _parse_http_content_type( http_content_type ) 

80 return _decode_content_charset_result( 

81 content, behaviors, profile, 

82 httpct_charset = httpct_charset, 

83 location = location, 

84 charset_supplement = charset_supplement )[ 0 ] 

85 

86 

87def decode_inform( # noqa: PLR0913 

88 content: _nomina.Content, /, *, 

89 behaviors: _BehaviorsArgument = _BEHAVIORS_DEFAULT, 

90 profile: _validation.ProfileArgument = _validation.PROFILE_TEXTUAL, 

91 mimetype_default: _nomina.MimetypeDefaultArgument = ( 

92 _MIMETYPE_DEFAULT_TEXTUAL ), 

93 http_content_type: _nomina.HttpContentTypeArgument = __.absent, 

94 location: _nomina.LocationArgument = __.absent, 

95 charset_supplement: _nomina.CharsetSupplementArgument = __.absent, 

96) -> DecodeInformResult: 

97 ''' Decodes bytes and returns supplemental inference metadata. ''' 

98 httpct_mimetype, httpct_charset = ( 

99 _parse_http_content_type( http_content_type ) ) 

100 text, charset_result = _decode_content_charset_result( 

101 content, behaviors, profile, 

102 httpct_charset = httpct_charset, 

103 location = location, 

104 charset_supplement = charset_supplement ) 

105 mimetype_result = _infer_mimetype( 

106 content, behaviors, 

107 mimetype_default = mimetype_default, 

108 httpct_mimetype = httpct_mimetype, 

109 location = location, 

110 charset = charset_result.charset ) 

111 linesep = _lineseparators.LineSeparators.detect_bytes( content ) 

112 return DecodeInformResult( 

113 text = text, 

114 charset = charset_result, 

115 mimetype = mimetype_result, 

116 linesep = linesep ) 

117 

118 

119def _attempt_decode_http_content_type( 

120 content: _nomina.Content, 

121 behaviors: _BehaviorsArgument, 

122 profile: _validation.ProfileArgument, /, *, 

123 httpct_charset: __.Absential[ __.typx.Optional[ str ] ], 

124 location: _nomina.LocationArgument, 

125) -> __.Absential[ tuple[ str, _CharsetResult ] ]: 

126 error = _exceptions.ContentDecodeImpossibility( location = location ) 

127 if httpct_charset is None: raise error 

128 if __.is_absent( httpct_charset ): return __.absent 

129 behaviors_ = __.dcls.replace( 

130 behaviors, trial_codecs = ( _CodecSpecifiers.FromInference, ) ) 

131 try: 

132 text, result = _charsets.attempt_decodes( 

133 content, 

134 behaviors = behaviors_, 

135 inference = httpct_charset, 

136 location = location ) 

137 except _exceptions.ContentDecodeFailure: return __.absent 

138 _validate_text( 

139 text, result.confidence, 

140 behaviors = behaviors, profile = profile, location = location ) 

141 return text, result 

142 

143 

144def _decode_content_charset_result( # noqa: PLR0913 

145 content: _nomina.Content, 

146 behaviors: _BehaviorsArgument, 

147 profile: _validation.ProfileArgument, /, *, 

148 httpct_charset: __.Absential[ __.typx.Optional[ str ] ], 

149 location: _nomina.LocationArgument, 

150 charset_supplement: _nomina.CharsetSupplementArgument, 

151) -> tuple[ str, _CharsetResult ]: 

152 if content == b'': 

153 return '', _CharsetResult( charset = 'utf-8', confidence = 1.0 ) 

154 charset: __.Absential[ str ] = __.absent 

155 result: __.Absential[ _CharsetResult ] = __.absent 

156 httpct_result: __.Absential[ tuple[ str, _CharsetResult ] ] = __.absent 

157 httpct_result = _attempt_decode_http_content_type( 

158 content, behaviors, profile, 

159 httpct_charset = httpct_charset, location = location ) 

160 if not __.is_absent( httpct_result ): return httpct_result 

161 behaviors_ = __.dcls.replace( 

162 behaviors, trial_decode = _BehaviorTristate.Never ) 

163 if behaviors.charset_detect: 

164 with __.ctxl.suppress( _exceptions.CharsetDetectFailure ): 

165 result = _detectors.detect_charset_confidence( 

166 content, 

167 behaviors = behaviors_, 

168 supplement = charset_supplement, 

169 location = location ) 

170 if ( result.charset 

171 and result.confidence >= behaviors.trial_decode_confidence 

172 ): charset = result.charset 

173 validator = __.funct.partial( 

174 _validate_text_in_decode_attempt, 

175 behaviors = behaviors, 

176 profile = profile, 

177 location = location ) 

178 return _charsets.attempt_decodes( 

179 content, 

180 behaviors = behaviors, 

181 inference = charset, 

182 supplement = charset_supplement, 

183 location = location, 

184 validator = validator ) 

185 

186 

187def _infer_mimetype( # noqa: PLR0913 

188 content: _nomina.Content, 

189 behaviors: _BehaviorsArgument, /, *, 

190 mimetype_default: _nomina.MimetypeDefaultArgument, 

191 httpct_mimetype: __.Absential[ str ], 

192 location: _nomina.LocationArgument, 

193 charset: __.typx.Optional[ str ], 

194) -> _MimetypeResult: 

195 charset_ = __.absent if charset is None else charset 

196 if ( not __.is_absent( httpct_mimetype ) 

197 and _mimetypes.is_textual_mimetype( httpct_mimetype ) 

198 ): 

199 return _MimetypeResult( mimetype = httpct_mimetype, confidence = 0.9 ) 

200 result: __.Absential[ _MimetypeResult ] = __.absent 

201 if not __.is_absent( location ): 

202 mimetype = _mimetypes.mimetype_from_location( location ) 

203 if ( not __.is_absent( mimetype ) 

204 and _mimetypes.is_textual_mimetype( mimetype ) 

205 ): 

206 return _MimetypeResult( mimetype = mimetype, confidence = 0.9 ) 

207 if behaviors.mimetype_detect: 

208 result = _detectors.detect_mimetype_confidence( 

209 content, 

210 behaviors = behaviors, 

211 default = mimetype_default, 

212 charset = charset_, 

213 location = location ) 

214 if __.is_absent( result ): 

215 return _MimetypeResult( mimetype = mimetype_default, confidence = 1.0 ) 

216 if _mimetypes.is_textual_mimetype( result.mimetype ): return result 

217 return _MimetypeResult( mimetype = mimetype_default, confidence = 1.0 ) 

218 

219 

220def _parse_http_content_type( 

221 http_content_type: _nomina.HttpContentTypeArgument 

222) -> tuple[ __.Absential[ str ], __.Absential[ __.typx.Optional[ str ] ] ]: 

223 if __.is_absent( http_content_type ): 

224 return __.absent, __.absent 

225 return _inference.parse_http_content_type( http_content_type ) 

226 

227 

228def _validate_text( 

229 text: str, confidence: float, /, *, 

230 behaviors: _BehaviorsArgument, 

231 profile: _validation.ProfileArgument, 

232 location: _nomina.LocationArgument, 

233) -> None: 

234 error = _exceptions.TextInvalidity( location = location ) 

235 should_validate = False 

236 match behaviors.text_validate: 

237 case _BehaviorTristate.Always: 

238 should_validate = True 

239 case _BehaviorTristate.AsNeeded: 

240 should_validate = confidence < behaviors.text_validate_confidence 

241 case _BehaviorTristate.Never: pass 

242 if should_validate and not profile( text ): raise error 

243 

244 

245def _validate_text_in_decode_attempt( 

246 text: str, result: _CharsetResult, /, *, 

247 behaviors: _BehaviorsArgument, 

248 profile: _validation.ProfileArgument, 

249 location: _nomina.LocationArgument, 

250) -> None: 

251 _validate_text( 

252 text, 0.0, 

253 behaviors = behaviors, 

254 profile = profile, 

255 location = location )