Skip to content

Deep scaning of squashfs containers #762

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
babenek wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Deep scaning of squashfs containers #762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/check.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ jobs:
run: |
banner="$(python -m credsweeper --banner | head -1)"
echo "banner = '${banner}'"
if [ "CredSweeper 1.14.2 crc32:36ab773c" != "${banner}" ]; then
if [ "CredSweeper 1.14.2 crc32:99a7460a" != "${banner}" ]; then
echo "Update the check for '${banner}'"
exit 1
fi
Expand Down
3 changes: 3 additions & 0 deletions .mypy.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,6 @@ ignore_missing_imports = True

[mypy-striprtf.*]
ignore_missing_imports = True

[mypy-PySquashfsImage.*]
ignore_missing_imports = True
5 changes: 5 additions & 0 deletions credsweeper/deep_scanner/deep_scanner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from .rpm_scanner import RpmScanner
from .rtf_scanner import RtfScanner
from .sqlite3_scanner import Sqlite3Scanner
from .squashfs_scanner import SquashfsScanner
from .strings_scanner import StringsScanner
from .tar_scanner import TarScanner
from .tmx_scanner import TmxScanner
Expand Down Expand Up @@ -54,6 +55,7 @@ class DeepScanner(
PptxScanner, #
RtfScanner, #
RpmScanner, #
SquashfsScanner, #
Sqlite3Scanner, #
StringsScanner, #
TarScanner, #
Expand Down Expand Up @@ -136,6 +138,9 @@ def get_deep_scanners(data: bytes, descriptor: Descriptor, depth: int) -> Tuple[
elif Util.is_sqlite3(data):
if 0 < depth:
deep_scanners.append(Sqlite3Scanner)
elif Util.is_squashfs(data):
if 0 < depth:
deep_scanners.append(SquashfsScanner)
elif Util.is_asn1(data):
deep_scanners.append(PkcsScanner)
elif Util.is_rtf(data):
Expand Down
52 changes: 52 additions & 0 deletions credsweeper/deep_scanner/squashfs_scanner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import logging
from abc import ABC
from typing import List, Optional

from PySquashfsImage import SquashFsImage

from credsweeper.credentials.candidate import Candidate
from credsweeper.deep_scanner.abstract_scanner import AbstractScanner
from credsweeper.file_handler.data_content_provider import DataContentProvider
from credsweeper.file_handler.file_path_extractor import FilePathExtractor
from credsweeper.utils.util import Util

logger = logging.getLogger(__name__)


class SquashfsScanner(AbstractScanner, ABC):
"""Implements squash file system scanning"""

def data_scan(
self, #
data_provider: DataContentProvider, #
depth: int, #
recursive_limit_size: int) -> Optional[List[Candidate]]:
"""Extracts files one by one from tar archive and launches data_scan"""
try:
candidates = []
with SquashFsImage.from_bytes(data_provider.data) as image:
for i in image:
# skip directory
if not i.is_file or i.is_symlink:
continue
logger.warning(f"{i.path}")
if FilePathExtractor.check_exclude_file(self.config, i.path):
continue
if 0 > recursive_limit_size - i.size:
logger.error(f"{i.name}: size {i.size}"
f" is over limit {recursive_limit_size} depth:{depth}")
continue
logger.warning(f"{i.path} {i.name}")
hsqs_content_provider = DataContentProvider(data=image.read_file(i.inode),
file_path=i.path,
file_type=Util.get_extension(i.path),
info=f"{data_provider.info}|HSQS:{i.path}")
# Nevertheless, use extracted data size
new_limit = recursive_limit_size - len(hsqs_content_provider.data)
logger.info(f"{i.name}: size {len(hsqs_content_provider.data)}")
hsqs_candidates = self.recursive_scan(hsqs_content_provider, depth, new_limit)
candidates.extend(hsqs_candidates)
return candidates
except Exception as hsqs_exc:
logger.error(f"{data_provider.file_path}:{hsqs_exc}")
return None
4 changes: 2 additions & 2 deletions credsweeper/secret/config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@
"exclude": {
"pattern": [],
"containers": [
".pak",
".aar",
".apk",
".bz2",
".class",
".gz",
".jar",
".img",
".lzma",
".rpm",
".tar",
Expand Down Expand Up @@ -44,7 +46,6 @@
".gif",
".gmo",
".ico",
".img",
".info",
".jpeg",
".jpg",
Expand All @@ -65,7 +66,6 @@
".ogg",
".ogv",
".ops",
".pak",
".png",
".psd",
".pyc",
Expand Down
10 changes: 10 additions & 0 deletions credsweeper/utils/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,16 @@ def is_rtf(data: Union[bytes, bytearray]):
return True
return False

@staticmethod
def is_squashfs(data):
"""According https://en.wikipedia.org/wiki/List_of_file_signatures - SQLite Database"""
if isinstance(data, (bytes, bytearray)) and data.startswith(b"hsqs") and b"\x04\x00\x00\x00" == data[28:32]:
# "Must be a power of two between 4096 (4k) and 1048576 (1 MiB)"
block_size = int.from_bytes(data[12:16], byteorder="little", signed=False)
if 0 == 0xFFF & block_size and 4096 <= block_size <= 1048576:
return True
return False

@staticmethod
def is_asn1(data: Union[bytes, bytearray]) -> int:
"""Only sequence type 0x30 and size correctness are checked
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ dependencies = [
"python-dateutil",
"python-docx",
"python-pptx",
"PySquashfsImage",
"PyYAML",
"rpmfile",
"striprtf",
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ numpy==2.3.3; python_version > '3.10'
odfpy==1.4.1
xlrd==2.0.2
striprtf==0.0.29
PySquashfsImage==0.9.0

# onnxruntime - ML engine
onnxruntime==1.23.2
Expand Down
4 changes: 2 additions & 2 deletions tests/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from pathlib import Path

# total number of files in test samples
SAMPLES_FILES_COUNT = 168
SAMPLES_FILES_COUNT = 169

# the lowest value of ML threshold is used to display possible lowest values
NEGLIGIBLE_ML_THRESHOLD = 0.0001
Expand All @@ -19,7 +19,7 @@
SAMPLES_POST_CRED_COUNT = 491

# archived credentials that are not found without --depth
SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 138
SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 142
SAMPLES_IN_DEEP_2 = SAMPLES_IN_DEEP_1 + 5
SAMPLES_IN_DEEP_3 = SAMPLES_IN_DEEP_2 + 4

Expand Down
201 changes: 201 additions & 0 deletions tests/data/depth_3_pedantic.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16627,5 +16627,206 @@
"entropy": 2.52164
}
]
},
{
"rule": "Github Classic Token",
"severity": "high",
"confidence": "strong",
"ml_probability": null,
"line_data_list": [
{
"line": "ghp_00000000000000000000000000000004WZ4EQ",
"line_num": 1,
"path": "/sample",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "ghp_00000000000000000000000000000004WZ4EQ",
"value_start": 0,
"value_end": 41,
"entropy": 1.56292
}
]
},
{
"rule": "Auth",
"severity": "medium",
"confidence": "moderate",
"ml_probability": 0.96,
"line_data_list": [
{
"line": "AUTH = (\"Basic YWRtaW46cGFyb2w0Mg==\")",
"line_num": 3,
"path": "/sample",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample|RAW",
"variable": "AUTH",
"variable_start": 0,
"variable_end": 4,
"value": "YWRtaW46cGFyb2w0Mg==",
"value_start": 15,
"value_end": 35,
"entropy": 4.12193
}
]
},
{
"rule": "Basic Authorization",
"severity": "medium",
"confidence": "strong",
"ml_probability": null,
"line_data_list": [
{
"line": "AUTH = (\"Basic YWRtaW46cGFyb2w0Mg==\")",
"line_num": 3,
"path": "/sample",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample|RAW",
"variable": "Basic",
"variable_start": 9,
"variable_end": 14,
"value": "YWRtaW46cGFyb2w0Mg==",
"value_start": 15,
"value_end": 35,
"entropy": 4.12193
}
]
},
{
"rule": "PEM Private Key",
"severity": "high",
"confidence": "strong",
"ml_probability": null,
"line_data_list": [
{
"line": "-----BEGIN PRIVATE KEY-----",
"line_num": 1,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "-----BEGIN PRIVATE KEY-----",
"value_start": 0,
"value_end": 27,
"entropy": 3.20029
},
{
"line": "MIIBVwIBADANBgkqhkiG9w0BAQEFAASCAUEwggE9AgEAAkEAr6IIv1xVwrhgoXp+",
"line_num": 2,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "MIIBVwIBADANBgkqhkiG9w0BAQEFAASCAUEwggE9AgEAAkEAr6IIv1xVwrhgoXp+",
"value_start": 0,
"value_end": 64,
"entropy": 4.48745
},
{
"line": "KOUDhLpzEPg7XaG2vfHaOB0++JzWnpvfii4BojVzrrkJjcHEA0975Ckp1bKp/swY",
"line_num": 3,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "KOUDhLpzEPg7XaG2vfHaOB0++JzWnpvfii4BojVzrrkJjcHEA0975Ckp1bKp/swY",
"value_start": 0,
"value_end": 64,
"entropy": 5.3007
},
{
"line": "r8qHZwIDAQABAkEAkFDJdUxO3+iW5jj7z2iW4oo+IJSIW2CVAzDmybMmWitRFIBZ",
"line_num": 4,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "r8qHZwIDAQABAkEAkFDJdUxO3+iW5jj7z2iW4oo+IJSIW2CVAzDmybMmWitRFIBZ",
"value_start": 0,
"value_end": 64,
"entropy": 5.07626
},
{
"line": "tkxP/yUtk8rysA/MmlFKAywnhsqrh/lKWcLPcQIhANjND990LTAtqe0vX6C/lt3p",
"line_num": 5,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "tkxP/yUtk8rysA/MmlFKAywnhsqrh/lKWcLPcQIhANjND990LTAtqe0vX6C/lt3p",
"value_start": 0,
"value_end": 64,
"entropy": 5.05836
},
{
"line": "PQbLinV/Uvv3TTn7ruhZAiEAz2NzQp9juvUz3ZVhgfer0zbWlLxT52dSAFzwAWWe",
"line_num": 6,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "PQbLinV/Uvv3TTn7ruhZAiEAz2NzQp9juvUz3ZVhgfer0zbWlLxT52dSAFzwAWWe",
"value_start": 0,
"value_end": 64,
"entropy": 5.03321
},
{
"line": "9b8CIQDN81ddza5Tsz1WbQdp3DcETdpERz0byz1Y0J/TMi0A2QIhAKlQVyyANorP",
"line_num": 7,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "9b8CIQDN81ddza5Tsz1WbQdp3DcETdpERz0byz1Y0J/TMi0A2QIhAKlQVyyANorP",
"value_start": 0,
"value_end": 64,
"entropy": 4.99173
},
{
"line": "X327VHUoQBbbgYrCynl0x+TV+3gYgBO/AiEAzIj5UNzfhW5pgJwMHz1G/nH6Ea0l",
"line_num": 8,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "X327VHUoQBbbgYrCynl0x+TV+3gYgBO/AiEAzIj5UNzfhW5pgJwMHz1G/nH6Ea0l",
"value_start": 0,
"value_end": 64,
"entropy": 5.28891
},
{
"line": "MSl+rkqtUopR0Ik=",
"line_num": 9,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "MSl+rkqtUopR0Ik=",
"value_start": 0,
"value_end": 16,
"entropy": 3.875
},
{
"line": "-----END PRIVATE KEY-----",
"line_num": 10,
"path": "/sample.pem",
"info": "FILE:./tests/samples/sample.hsqs|HSQS:/sample.pem|RAW",
"variable": null,
"variable_start": -2,
"variable_end": -2,
"value": "-----END PRIVATE KEY-----",
"value_start": 0,
"value_end": 25,
"entropy": 3.04489
}
]
}
]
Binary file added tests/samples/sample.hsqs
View file
Open in desktop
Binary file not shown.
2 changes: 1 addition & 1 deletion tests/test_app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,7 @@ def test_depth_p(self) -> None:
cvs_checksum = hashlib.md5(f.read()).digest()
checksum = bytes(a ^ b for a, b in zip(checksum, cvs_checksum))
# update the checksum manually and keep line endings in the samples as is (git config core.autocrlf false)
self.assertEqual("4a12fce9d4c17e6b3aaef0f4be070225", binascii.hexlify(checksum).decode())
self.assertEqual("080e6166edd7cffc5564eb96a2ce7c68", binascii.hexlify(checksum).decode())
normal_report = []
sorted_report = []
with tempfile.TemporaryDirectory() as tmp_dir:
Expand Down