Skip to content

Commit 047113f

Browse files
babenekbabenek
committed
[skip actions] [hsqs] 2026-01-01T09:56:39+02:00
1 parent 22bc792 commit 047113f

File tree

12 files changed

+279
-6
lines changed

12 files changed

+279
-6
lines changed

‎.github/workflows/check.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ jobs:
9292
run: |
9393
banner="$(python -m credsweeper --banner | head -1)"
9494
echo "banner = '${banner}'"
95-
if [ "CredSweeper 1.14.2 crc32:36ab773c" != "${banner}" ]; then
95+
if [ "CredSweeper 1.14.2 crc32:99a7460a" != "${banner}" ]; then
9696
echo "Update the check for '${banner}'"
9797
exit 1
9898
fi

‎.mypy.ini‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,6 @@ ignore_missing_imports = True
5858

5959
[mypy-striprtf.*]
6060
ignore_missing_imports = True
61+
62+
[mypy-PySquashfsImage.*]
63+
ignore_missing_imports = True

‎credsweeper/deep_scanner/deep_scanner.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from .rpm_scanner import RpmScanner
2626
from .rtf_scanner import RtfScanner
2727
from .sqlite3_scanner import Sqlite3Scanner
28+
from .squashfs_scanner import SquashfsScanner
2829
from .strings_scanner import StringsScanner
2930
from .tar_scanner import TarScanner
3031
from .tmx_scanner import TmxScanner
@@ -54,6 +55,7 @@ class DeepScanner(
5455
PptxScanner, #
5556
RtfScanner, #
5657
RpmScanner, #
58+
SquashfsScanner, #
5759
Sqlite3Scanner, #
5860
StringsScanner, #
5961
TarScanner, #
@@ -136,6 +138,9 @@ def get_deep_scanners(data: bytes, descriptor: Descriptor, depth: int) -> Tuple[
136138
elif Util.is_sqlite3(data):
137139
if 0 < depth:
138140
deep_scanners.append(Sqlite3Scanner)
141+
elif Util.is_squashfs(data):
142+
if 0 < depth:
143+
deep_scanners.append(SquashfsScanner)
139144
elif Util.is_asn1(data):
140145
deep_scanners.append(PkcsScanner)
141146
elif Util.is_rtf(data):

‎credsweeper/deep_scanner/squashfs_scanner.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import logging
2+
from abc import ABC
3+
from typing import List, Optional
4+
5+
from PySquashfsImage import SquashFsImage
6+
7+
from credsweeper.credentials.candidate import Candidate
8+
from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
9+
from credsweeper.file_handler.data_content_provider import DataContentProvider
10+
from credsweeper.file_handler.file_path_extractor import FilePathExtractor
11+
from credsweeper.utils.util import Util
12+
13+
logger = logging.getLogger(__name__)
14+
15+
16+
class SquashfsScanner(AbstractScanner, ABC):
17+
"""Implements squash file system scanning"""
18+
19+
def data_scan(
20+
self, #
21+
data_provider: DataContentProvider, #
22+
depth: int, #
23+
recursive_limit_size: int) -> Optional[List[Candidate]]:
24+
"""Extracts files one by one from tar archive and launches data_scan"""
25+
try:
26+
candidates = []
27+
with SquashFsImage.from_bytes(data_provider.data) as image:
28+
for i in image:
29+
# skip directory
30+
if not i.is_file or i.is_symlink:
31+
continue
32+
logger.warning(f"{i.path}")
33+
if FilePathExtractor.check_exclude_file(self.config, i.path):
34+
continue
35+
if 0 > recursive_limit_size - i.size:
36+
logger.error(f"{i.name}: size {i.size}"
37+
f" is over limit {recursive_limit_size} depth:{depth}")
38+
continue
39+
logger.warning(f"{i.path} {i.name}")
40+
hsqs_content_provider = DataContentProvider(data=image.read_file(i.inode),
41+
file_path=i.path,
42+
file_type=Util.get_extension(i.path),
43+
info=f"{data_provider.info}|HSQS:{i.path}")
44+
# Nevertheless, use extracted data size
45+
new_limit = recursive_limit_size - len(hsqs_content_provider.data)
46+
logger.info(f"{i.name}: size {len(hsqs_content_provider.data)}")
47+
hsqs_candidates = self.recursive_scan(hsqs_content_provider, depth, new_limit)
48+
candidates.extend(hsqs_candidates)
49+
return candidates
50+
except Exception as hsqs_exc:
51+
logger.error(f"{data_provider.file_path}:{hsqs_exc}")
52+
return None

‎credsweeper/secret/config.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@
22
"exclude": {
33
"pattern": [],
44
"containers": [
5+
".pak",
56
".aar",
67
".apk",
78
".bz2",
89
".class",
910
".gz",
1011
".jar",
12+
".img",
1113
".lzma",
1214
".rpm",
1315
".tar",
@@ -44,7 +46,6 @@
4446
".gif",
4547
".gmo",
4648
".ico",
47-
".img",
4849
".info",
4950
".jpeg",
5051
".jpg",
@@ -65,7 +66,6 @@
6566
".ogg",
6667
".ogv",
6768
".ops",
68-
".pak",
6969
".png",
7070
".psd",
7171
".pyc",

‎credsweeper/utils/util.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,16 @@ def is_rtf(data: Union[bytes, bytearray]):
364364
return True
365365
return False
366366

367+
@staticmethod
368+
def is_squashfs(data):
369+
"""According https://en.wikipedia.org/wiki/List_of_file_signatures - SQLite Database"""
370+
if isinstance(data, (bytes, bytearray)) and data.startswith(b"hsqs") and b"\x04\x00\x00\x00" == data[28:32]:
371+
# "Must be a power of two between 4096 (4k) and 1048576 (1 MiB)"
372+
block_size = int.from_bytes(data[12:16], byteorder="little", signed=False)
373+
if 0 == 0xFFF & block_size and 4096 <= block_size <= 1048576:
374+
return True
375+
return False
376+
367377
@staticmethod
368378
def is_asn1(data: Union[bytes, bytearray]) -> int:
369379
"""Only sequence type 0x30 and size correctness are checked

‎pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ dependencies = [
2424
"python-dateutil",
2525
"python-docx",
2626
"python-pptx",
27+
"PySquashfsImage",
2728
"PyYAML",
2829
"rpmfile",
2930
"striprtf",

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ numpy==2.3.3; python_version > '3.10'
2222
odfpy==1.4.1
2323
xlrd==2.0.2
2424
striprtf==0.0.29
25+
PySquashfsImage==0.9.0
2526

2627
# onnxruntime - ML engine
2728
onnxruntime==1.23.2

‎tests/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from pathlib import Path
22

33
# total number of files in test samples
4-
SAMPLES_FILES_COUNT = 168
4+
SAMPLES_FILES_COUNT = 169
55

66
# the lowest value of ML threshold is used to display possible lowest values
77
NEGLIGIBLE_ML_THRESHOLD = 0.0001
@@ -19,7 +19,7 @@
1919
SAMPLES_POST_CRED_COUNT = 491
2020

2121
# archived credentials that are not found without --depth
22-
SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 138
22+
SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 142
2323
SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 5
2424
SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 4
2525

‎tests/data/depth_3_pedantic.json‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16627,5 +16627,206 @@
1662716627
"entropy": 2.52164
1662816628
}
1662916629
]
16630+
},
16631+
{
16632+
"rule": "Github Classic Token",
16633+
"severity": "high",
16634+
"confidence": "strong",
16635+
"ml_probability": null,
16636+
"line_data_list": [
16637+
{
16638+
"line": "ghp_00000000000000000000000000000004WZ4EQ",
16639+
"line_num": 1,
16640+
"path": "/sample",
16641+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample|RAW",
16642+
"variable": null,
16643+
"variable_start": -2,
16644+
"variable_end": -2,
16645+
"value": "ghp_00000000000000000000000000000004WZ4EQ",
16646+
"value_start": 0,
16647+
"value_end": 41,
16648+
"entropy": 1.56292
16649+
}
16650+
]
16651+
},
16652+
{
16653+
"rule": "Auth",
16654+
"severity": "medium",
16655+
"confidence": "moderate",
16656+
"ml_probability": 0.96,
16657+
"line_data_list": [
16658+
{
16659+
"line": "AUTH = (\"Basic YWRtaW46cGFyb2w0Mg==\")",
16660+
"line_num": 3,
16661+
"path": "/sample",
16662+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample|RAW",
16663+
"variable": "AUTH",
16664+
"variable_start": 0,
16665+
"variable_end": 4,
16666+
"value": "YWRtaW46cGFyb2w0Mg==",
16667+
"value_start": 15,
16668+
"value_end": 35,
16669+
"entropy": 4.12193
16670+
}
16671+
]
16672+
},
16673+
{
16674+
"rule": "Basic Authorization",
16675+
"severity": "medium",
16676+
"confidence": "strong",
16677+
"ml_probability": null,
16678+
"line_data_list": [
16679+
{
16680+
"line": "AUTH = (\"Basic YWRtaW46cGFyb2w0Mg==\")",
16681+
"line_num": 3,
16682+
"path": "/sample",
16683+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample|RAW",
16684+
"variable": "Basic",
16685+
"variable_start": 9,
16686+
"variable_end": 14,
16687+
"value": "YWRtaW46cGFyb2w0Mg==",
16688+
"value_start": 15,
16689+
"value_end": 35,
16690+
"entropy": 4.12193
16691+
}
16692+
]
16693+
},
16694+
{
16695+
"rule": "PEM Private Key",
16696+
"severity": "high",
16697+
"confidence": "strong",
16698+
"ml_probability": null,
16699+
"line_data_list": [
16700+
{
16701+
"line": "-----BEGIN PRIVATE KEY-----",
16702+
"line_num": 1,
16703+
"path": "/sample.pem",
16704+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16705+
"variable": null,
16706+
"variable_start": -2,
16707+
"variable_end": -2,
16708+
"value": "-----BEGIN PRIVATE KEY-----",
16709+
"value_start": 0,
16710+
"value_end": 27,
16711+
"entropy": 3.20029
16712+
},
16713+
{
16714+
"line": "MIIBVwIBADANBgkqhkiG9w0BAQEFAASCAUEwggE9AgEAAkEAr6IIv1xVwrhgoXp+",
16715+
"line_num": 2,
16716+
"path": "/sample.pem",
16717+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16718+
"variable": null,
16719+
"variable_start": -2,
16720+
"variable_end": -2,
16721+
"value": "MIIBVwIBADANBgkqhkiG9w0BAQEFAASCAUEwggE9AgEAAkEAr6IIv1xVwrhgoXp+",
16722+
"value_start": 0,
16723+
"value_end": 64,
16724+
"entropy": 4.48745
16725+
},
16726+
{
16727+
"line": "KOUDhLpzEPg7XaG2vfHaOB0++JzWnpvfii4BojVzrrkJjcHEA0975Ckp1bKp/swY",
16728+
"line_num": 3,
16729+
"path": "/sample.pem",
16730+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16731+
"variable": null,
16732+
"variable_start": -2,
16733+
"variable_end": -2,
16734+
"value": "KOUDhLpzEPg7XaG2vfHaOB0++JzWnpvfii4BojVzrrkJjcHEA0975Ckp1bKp/swY",
16735+
"value_start": 0,
16736+
"value_end": 64,
16737+
"entropy": 5.3007
16738+
},
16739+
{
16740+
"line": "r8qHZwIDAQABAkEAkFDJdUxO3+iW5jj7z2iW4oo+IJSIW2CVAzDmybMmWitRFIBZ",
16741+
"line_num": 4,
16742+
"path": "/sample.pem",
16743+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16744+
"variable": null,
16745+
"variable_start": -2,
16746+
"variable_end": -2,
16747+
"value": "r8qHZwIDAQABAkEAkFDJdUxO3+iW5jj7z2iW4oo+IJSIW2CVAzDmybMmWitRFIBZ",
16748+
"value_start": 0,
16749+
"value_end": 64,
16750+
"entropy": 5.07626
16751+
},
16752+
{
16753+
"line": "tkxP/yUtk8rysA/MmlFKAywnhsqrh/lKWcLPcQIhANjND990LTAtqe0vX6C/lt3p",
16754+
"line_num": 5,
16755+
"path": "/sample.pem",
16756+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16757+
"variable": null,
16758+
"variable_start": -2,
16759+
"variable_end": -2,
16760+
"value": "tkxP/yUtk8rysA/MmlFKAywnhsqrh/lKWcLPcQIhANjND990LTAtqe0vX6C/lt3p",
16761+
"value_start": 0,
16762+
"value_end": 64,
16763+
"entropy": 5.05836
16764+
},
16765+
{
16766+
"line": "PQbLinV/Uvv3TTn7ruhZAiEAz2NzQp9juvUz3ZVhgfer0zbWlLxT52dSAFzwAWWe",
16767+
"line_num": 6,
16768+
"path": "/sample.pem",
16769+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16770+
"variable": null,
16771+
"variable_start": -2,
16772+
"variable_end": -2,
16773+
"value": "PQbLinV/Uvv3TTn7ruhZAiEAz2NzQp9juvUz3ZVhgfer0zbWlLxT52dSAFzwAWWe",
16774+
"value_start": 0,
16775+
"value_end": 64,
16776+
"entropy": 5.03321
16777+
},
16778+
{
16779+
"line": "9b8CIQDN81ddza5Tsz1WbQdp3DcETdpERz0byz1Y0J/TMi0A2QIhAKlQVyyANorP",
16780+
"line_num": 7,
16781+
"path": "/sample.pem",
16782+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16783+
"variable": null,
16784+
"variable_start": -2,
16785+
"variable_end": -2,
16786+
"value": "9b8CIQDN81ddza5Tsz1WbQdp3DcETdpERz0byz1Y0J/TMi0A2QIhAKlQVyyANorP",
16787+
"value_start": 0,
16788+
"value_end": 64,
16789+
"entropy": 4.99173
16790+
},
16791+
{
16792+
"line": "X327VHUoQBbbgYrCynl0x+TV+3gYgBO/AiEAzIj5UNzfhW5pgJwMHz1G/nH6Ea0l",
16793+
"line_num": 8,
16794+
"path": "/sample.pem",
16795+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16796+
"variable": null,
16797+
"variable_start": -2,
16798+
"variable_end": -2,
16799+
"value": "X327VHUoQBbbgYrCynl0x+TV+3gYgBO/AiEAzIj5UNzfhW5pgJwMHz1G/nH6Ea0l",
16800+
"value_start": 0,
16801+
"value_end": 64,
16802+
"entropy": 5.28891
16803+
},
16804+
{
16805+
"line": "MSl+rkqtUopR0Ik=",
16806+
"line_num": 9,
16807+
"path": "/sample.pem",
16808+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16809+
"variable": null,
16810+
"variable_start": -2,
16811+
"variable_end": -2,
16812+
"value": "MSl+rkqtUopR0Ik=",
16813+
"value_start": 0,
16814+
"value_end": 16,
16815+
"entropy": 3.875
16816+
},
16817+
{
16818+
"line": "-----END PRIVATE KEY-----",
16819+
"line_num": 10,
16820+
"path": "/sample.pem",
16821+
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
16822+
"variable": null,
16823+
"variable_start": -2,
16824+
"variable_end": -2,
16825+
"value": "-----END PRIVATE KEY-----",
16826+
"value_start": 0,
16827+
"value_end": 25,
16828+
"entropy": 3.04489
16829+
}
16830+
]
1663016831
}
1663116832
]

0 commit comments

Comments
 (0)