From 558c90807b09b16757c88f2e50bdbd123d425f37 Mon Sep 17 00:00:00 2001 From: Leonard Kugis Date: Fri, 4 Sep 2026 04:01:49 +0200 Subject: Initial commit --- .env.example | 2 + .gitignore | 110 +++++ Dockerfile | 18 + README.md | 236 +++++++++++ app.py | 1019 ++++++++++++++++++++++++++++++++++++++++++++++ archival.py | 860 ++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 22 + squashr-tags-export | 36 ++ squashr-tags-integration | 40 ++ templates/index.html | 534 ++++++++++++++++++++++++ test_integration.py | 83 ++++ 11 files changed, 2960 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100755 app.py create mode 100644 archival.py create mode 100644 docker-compose.yml create mode 100755 squashr-tags-export create mode 100755 squashr-tags-integration create mode 100644 templates/index.html create mode 100644 test_integration.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..41c29f0 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +MEDMAN_PORT=8080 +MEDMAN_W= \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0bb0d5e --- /dev/null +++ b/.gitignore @@ -0,0 +1,110 @@ +*.pyc +__pycache__/ +*.egg-info/ +.venv/ +.env +data0 +data1 +src0 +src1 +db + +# Created by https://www.toptal.com/developers/gitignore/api/vim,linux,windows,macos +# Edit at https://www.toptal.com/developers/gitignore?templates=vim,linux,windows,macos + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/vim,linux,windows,macos + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7963438 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.11-slim-bookworm + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + squashfs-tools \ + cryptsetup-bin \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir flask pillow jinja2 + +WORKDIR /app +COPY . . + +# Base directories (mount subdirs via docker-compose) +RUN mkdir -p /app/db /app/srcs /app/dats /app/imports + +EXPOSE 8080 +CMD ["python", "app.py"] \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..df1f37e --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +# MedMan — Self-Hosted Media Manager with Tagging & squashr Integration + +MedMan is a web-based media manager designed to work alongside +[squashr](https://git.kug.is/squashr.git) (SquashFS/OverlayFS/LUKS backup tool). + +## Overview + +``` +1. Drop files into source directories (or upload via web UI) + ↓ +2. MedMan scans/ingests → files are distributed into data directories + using a full-hash directory hierarchy (every 2 hex chars = one level) + ↓ +3. Browse & tag files through the web interface + ↓ +4. Export tags → archival.py packs them into SquashFS containers + ↓ +5. Burn containers to CD/DVD/Blu-Ray or archive offline + ↓ +6. Offline use: mount container, start MedMan, import index.json → + browse & search media with full tag metadata +``` + +## Configuration + +MedMan is configured primarily through **Docker volume mounts**. Environment +variables are minimal: + +| Variable | Description | Default | +|---|---|---| +| `MEDMAN_PORT` | HTTP port | `8080` | +| `MEDMAN_HOST` | Bind address | `0.0.0.0` | +| `MEDMAN_W` | Comma-separated weights for data-dir distribution | equal weight | + +### Directory Structure + +All directories are inside `/app` in the container and wired via +`docker-compose.yml` mounts: + +``` +/app/ +├── srcs/ ← source directories (immediate subdirs) +│ ├── src0/ ← mount: ./src0:/app/srcs/src0 +│ └── src1/ ← mount: ./src1:/app/srcs/src1 +├── dats/ ← data directories (immediate subdirs) +│ ├── data0/ ← mount: ./data0:/app/dats/data0 +│ └── data1/ ← mount: ./data1:/app/dats/data1 +├── imports/ ← auto-import directories (immediate subdirs, read-only) +│ └── my-archive/ ← mount: /mnt/archive:/app/imports/my-archive:ro +└── db/ ← SQLite database + thumbnails (auto-created) +``` + +**Key principle**: Every immediate subdirectory of `srcs/`, `dats/`, and +`imports/` is automatically discovered. No comma-separated path lists in +environment variables. + +### Data Distribution Weights + +`MEDMAN_W` controls how new files are distributed across data directories: + +```bash +# 3 data dirs — dir 0 gets 50%, dirs 1 and 2 get 25% each +MEDMAN_W=2,1,1 +``` + +If not set, distribution is equal across all data directories. + +## Hash-Based Storage + +Every file is SHA-256 hashed. All 32 byte-pairs of the 64-character hex +hash become directory levels: + +``` +Hash: a1b2c3d4e5f6...a1b2c3d4e5f6 (64 chars) +Path: data0/a1/b2/c3/d4/e5/f6/.../a1/b2/c3/d4/e5/f6 +``` + +- 32 directory levels + filename (last hex pair) +- Automatic deduplication via content hashing +- Distribution across data directories according to `MEDMAN_W` weights +- No `rel` column stored in the database — the path is always deterministically + computed from the hash + +## Quickstart (Docker) + +```bash +cd medman +docker compose up -d +``` + +MedMan is available at `http://localhost:8080`. + +Drop files into `src0/` or `src1/` — they are automatically scanned and +distributed into the data directories on startup. Upload is also supported +via the web UI. + +## Auto-Import on Startup + +Directories mounted under `/app/imports/` are scanned on startup (and on +each manual scan). Each subdirectory may contain: + +- `index.json` — a MedMan export file with tags and file references +- Files in the canonical hash-prefix hierarchy (same layout as data dirs) + +All tags and file registrations from `index.json` are imported automatically +if the referenced files exist on disk. Import directories receive zero +distribution weight (new files are never placed there). + +Example: Mount a read-only SquashFS container containing an `index.json` and +canonical file paths: + +```yaml +# docker-compose.yml +volumes: + - /mnt/my-archive:/app/imports/my-archive:ro +``` + +## Features + +- **Source directories**: files auto-ingested on startup, upload via UI +- **Manual scan button**: re-scan source and import directories anytime +- **Hash-based storage**: content-addressable, no duplicates +- **Multiple data directories**: weighted random distribution +- **Move**: relocate individual files or selections between data dirs +- **Tag system**: plain labels, no color coding +- **Tag search**: combined AND search across multiple tags +- **Bulk operations**: select-all, batch tag/untag/move/delete +- **Previous/Next navigation**: arrow keys or buttons in detail view +- **Checkbox selection**: click the checkbox on any thumbnail to toggle +- **Export/Import**: tags as JSON (export/import) +- **archival.py**: create SquashFS + LUKS containers directly from MedMan +- **Container mount API**: mount SquashFS containers as additional data dirs + +## Database Schema + +```sql +tags(id, name, ts) +files(hash PK, didx, size, mt, w, h, dur, ts) +ftags(id, hash FK→files, tid FK→tags, UNIQUE(hash, tid)) +``` + +`didx` is the index into the `dats/` subdirectory list. The on-disk path +is always `dats//`. + +## API + +| Endpoint | Method | Description | +|---|---|---| +| `/api/files` | GET | All files (`?tag=` filter) | +| `/api/files/` | DELETE | Delete file | +| `/api/files//move` | POST | `{"didx":N}` move to data dir | +| `/api/files//tags` | GET/POST | Get or add tags | +| `/api/files//tags/` | DELETE | Remove tag from file | +| `/api/tags` | GET/POST | List or create tags | +| `/api/tags/` | DELETE | Delete tag | +| `/api/tags/export` | GET | Full tag export (`?tags=a,b` AND filter) | +| `/api/tags/import` | POST | Import tag JSON | +| `/api/search?tags=a,b` | GET | AND tag search | +| `/api/upload` | POST | Upload file(s) | +| `/api/scan` | POST | Scan sources + imports | +| `/api/dat` | GET | Data directory stats | +| `/api/containers/mount` | POST | Mount container dir as data dir | +| `/api/containers/mount` | DELETE | Unmount container dir | +| `/thumb/` | GET | JPEG thumbnail | +| `/raw/` | GET | Original file | + +## Export Format (index.json) + +```json +{ + "ts": "2025-01-15T12:00:00+00:00", + "tags": [{"id": 1, "name": "example"}], + "files": { + "a1b2c3d4...": {"didx": 0, "tags": ["example"]} + } +} +``` + +The on-disk path for each file is deterministic: `h2rel(hash)` → 32 hex-pair +directory levels. No `rel` field is needed. + +## archival.py — Creating SquashFS Containers + +```bash +# Build containers via MedMan API +python3 archival.py --medman-url http://localhost:8080 \ + --dat /app/data0 /app/data1 \ + --tags vacation 2024 \ + --container-size 4700372992 \ + --output-prefix ./archive + +# Build with LUKS encryption +python3 archival.py --db /app/db/medman.db \ + --dat /app/data0 /app/data1 \ + --tags photos \ + --container-size 25000000000 \ + --output-prefix ./photos \ + --cryptsetup --key-file /path/to/keyfile + +# Direct DB access (no running MedMan needed) +python3 archival.py --db /app/db/medman.db \ + --dat /app/data0 /app/data1 \ + --container-size 10000000000 \ + --output-prefix ./offline +``` + +The script uses `tar → sqfstar` piping (no staging directories, no temp +files beyond a single index.json) — exactly like squashr's +`build_squash_image_tar_sqfstar`. + +### Container Size Margin + +A margin of 1 MiB is reserved per container for the `index.json` and +filesystem metadata overhead. Adjust with `--margin` if needed. + +### Filter Options + +| Option | Description | +|---|---| +| `--tags a b` | Only files with ALL given tags (AND) | +| `--hashes a1b2...` | Specific hashes | +| `--hashes-from-file` | Read hashes from file | +| `--from-date 2024-01-01` | Modified on or after | +| `--to-date 2024-12-31` | Modified on or before | +| `--mime-type video` | Filter by media type | + +## squashr Integration + +MedMan containers produced by `archival.py` are standard SquashFS images +with an `index.json` at the root plus the canonical hash hierarchy. They +can be mounted and imported into any MedMan instance via the auto-import +feature or the container mount API. + +## License + +GNU AGPLv3 diff --git a/app.py b/app.py new file mode 100755 index 0000000..02f74ad --- /dev/null +++ b/app.py @@ -0,0 +1,1019 @@ +#!/usr/bin/env python3 +""" +MedMan — Self-hosted media manager with tagging & squashr integration. + +Source directories: files are ingested on startup / manual scan trigger. + All immediate subdirectories of /app/srcs are treated as source dirs. + +Data directories: files stored under full content-hash prefix paths + (2 hex chars = 1 byte per directory level, entire hash used). + All immediate subdirectories of /app/dats are treated as data dirs. + +Import directories: mounted under /app/imports. Each subdirectory may + contain an index.json at its root (MedMan export format) and files in + the canonical hash-prefix hierarchy. Tags and file registrations are + imported automatically on startup and re-checked on manual scan. + +Tag system (no color coding). Upload via web UI. Move files between data dirs. +""" + +import os, sys, io, json, hashlib, sqlite3, subprocess +import shutil, tempfile, secrets +from datetime import datetime, timezone +from typing import Optional + +from flask import Flask, render_template, request, jsonify, send_file, abort +from werkzeug.utils import secure_filename + +# ============================================================================ +# Configuration +# ============================================================================ + +PORT = int(os.environ.get("MEDMAN_PORT", "8080")) +HOST = os.environ.get("MEDMAN_HOST", "0.0.0.0") +DBDIR = os.path.join(os.getcwd(), "db") + +# Source directories: all immediate subdirs of /app/srcs +_SRC_ROOT = os.path.join(os.getcwd(), "srcs") +SRC: list[str] = [] +if os.path.isdir(_SRC_ROOT): + SRC = sorted([ + os.path.join(_SRC_ROOT, d) for d in os.listdir(_SRC_ROOT) + if os.path.isdir(os.path.join(_SRC_ROOT, d)) + ]) +if not SRC: + SRC = [os.path.join(os.getcwd(), "src0"), os.path.join(os.getcwd(), "src1")] + +# Data directories: all immediate subdirs of /app/dats +_DAT_ROOT = os.path.join(os.getcwd(), "dats") +DAT: list[str] = [] +if os.path.isdir(_DAT_ROOT): + DAT = sorted([ + os.path.join(_DAT_ROOT, d) for d in os.listdir(_DAT_ROOT) + if os.path.isdir(os.path.join(_DAT_ROOT, d)) + ]) +if not DAT: + DAT = [os.path.join(os.getcwd(), "data0"), os.path.join(os.getcwd(), "data1")] + +# Import directories: all immediate subdirs of /app/imports +# Each may contain an index.json + canonical hash-prefix hierarchy +_IMPORT_ROOT = os.path.join(os.getcwd(), "imports") +IMPORTS: list[str] = [] +if os.path.isdir(_IMPORT_ROOT): + IMPORTS = sorted([ + os.path.join(_IMPORT_ROOT, d) for d in os.listdir(_IMPORT_ROOT) + if os.path.isdir(os.path.join(_IMPORT_ROOT, d)) + ]) + +# Weights (comma-separated floats; default equal) +_w = os.environ.get("MEDMAN_W", "") +W: list[float] = [] +if _w.strip(): + W = [float(x.strip()) for x in _w.split(",") if x.strip()] +if not W: + W = [1.0] * len(DAT) +if len(W) != len(DAT): + raise RuntimeError("MEDMAN_W must have same count as data dirs") +tw = sum(W) +W = [x / tw for x in W] +CUM: list[float] = [] +c = 0.0 +for x in W: + c += x + CUM.append(c) + +DB = os.path.join(DBDIR, "medman.db") +TH = os.path.join(DBDIR, "thumbs") + +for d in [DBDIR, TH] + DAT + SRC + IMPORTS: + os.makedirs(d, exist_ok=True) + +app = Flask(__name__) + +# ============================================================================ +# Hash helpers +# ============================================================================ + +def h_file(ap: str) -> str: + h = hashlib.sha256() + with open(ap, "rb") as f: + while True: + b = f.read(65536) + if not b: + break + h.update(b) + return h.hexdigest() + + +def h2rel(hx: str) -> str: + """Every 2 hex chars → one directory level (32 levels + filename).""" + pairs = [hx[i:i + 2] for i in range(0, len(hx), 2)] + return "/".join(pairs) + + +def rel_for_disk(hx: str, didx: int) -> str: + """Full on-disk relative path for a given hash and data dir index.""" + return h2rel(hx) + + +def pick() -> tuple[int, str]: + r = secrets.randbelow(2**31) / (2**31) + for i, cu in enumerate(CUM): + if r <= cu: + return i, DAT[i] + return len(DAT) - 1, DAT[-1] + + +# ============================================================================ +# Database +# ============================================================================ + +def gdb(): + c = sqlite3.connect(DB) + c.row_factory = sqlite3.Row + c.execute("PRAGMA journal_mode=WAL") + c.execute("PRAGMA foreign_keys=ON") + return c + + +def idb(): + db = gdb() + db.executescript(""" + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + ts TEXT DEFAULT (datetime('now')) + ); + + -- rel removed: it is always deterministically h2rel(hash) + CREATE TABLE IF NOT EXISTS files ( + hash TEXT PRIMARY KEY, + didx INTEGER NOT NULL, + size INTEGER, + mt TEXT, + w INTEGER, + h INTEGER, + dur REAL, + ts TEXT DEFAULT (datetime('now')) + ); + + CREATE TABLE IF NOT EXISTS ftags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT NOT NULL, + tid INTEGER NOT NULL, + FOREIGN KEY (hash) REFERENCES files(hash) ON DELETE CASCADE, + FOREIGN KEY (tid) REFERENCES tags(id) ON DELETE CASCADE, + UNIQUE(hash, tid) + ); + CREATE INDEX IF NOT EXISTS ix_ft_h ON ftags(hash); + CREATE INDEX IF NOT EXISTS ix_ft_t ON ftags(tid); + CREATE INDEX IF NOT EXISTS ix_ta_n ON tags(name); + """) + # Drop 'rel' column if it still exists (migration from older schema) + cols = [r[1] for r in db.execute("PRAGMA table_info(files)").fetchall()] + if "rel" in cols: + db.execute("ALTER TABLE files DROP COLUMN rel") + db.commit() + db.close() + + +idb() + +# ============================================================================ +# Helpers +# ============================================================================ + +VE = {".mp4", ".mkv", ".webm", ".avi", ".mov", ".m4v", ".wmv", ".flv", ".ogv"} +IE = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".tiff", ".ico"} +AE = {".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a", ".wma", ".opus"} +DE = {".pdf", ".epub", ".txt", ".md", ".rst"} +TS = (320, 240) + + +def _mt_from_content(ap: str) -> str: + """Determine media type from file content (magic bytes / ffprobe).""" + try: + r = subprocess.run( + ["file", "-b", "--mime-type", ap], + capture_output=True, text=True, timeout=5, + ) + mt = r.stdout.strip() + if mt.startswith("video/"): return "video" + if mt.startswith("image/"): return "image" + if mt.startswith("audio/"): return "audio" + if mt.startswith("application/pdf"): return "doc" + if mt.startswith("text/"): return "doc" + except Exception: + pass + try: + r = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "format=format_name", + "-of", "csv=p=0", ap], + capture_output=True, text=True, timeout=10, + ) + fmts = r.stdout.strip().lower() + if any(x in fmts for x in ("mp4", "mov", "matroska", "avi")): return "video" + if any(x in fmts for x in ("mp3", "aac", "flac", "wav", "ogg")): return "audio" + except Exception: + pass + try: + from PIL import Image + Image.open(ap) + return "image" + except Exception: + pass + return "other" + + +def thumb(ap: str) -> Optional[bytes]: + """Generate thumbnail.""" + try: + from PIL import Image + im = Image.open(ap) + im.thumbnail(TS) + b = io.BytesIO() + im.save(b, format="JPEG", quality=75) + return b.getvalue() + except Exception: + pass + try: + r = subprocess.run([ + "ffmpeg", "-y", "-loglevel", "error", "-i", ap, + "-ss", "5", "-vframes", "1", + "-vf", f"scale={TS[0]}:{TS[1]}:force_original_aspect_ratio=decrease," + f"pad={TS[0]}:{TS[1]}:(ow-iw)/2:(oh-ih)/2", + "-f", "image2", "pipe:1", + ], capture_output=True, timeout=30) + if r.returncode == 0 and r.stdout: + return r.stdout + except Exception: + pass + return None + + +def probe(ap: str) -> dict: + try: + r = subprocess.run([ + "ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_entries", "stream=width,height,duration", "-of", "json", ap, + ], capture_output=True, timeout=15) + if r.returncode == 0: + d = json.loads(r.stdout) + if d.get("streams"): + s = d["streams"][0] + return {"w": s.get("width"), "h": s.get("height"), + "dur": s.get("duration")} + except Exception: + pass + return {} + + +def _resolve_file_path(hx: str, didx: int) -> str: + """Given a hash and data-dir index, compute the absolute file path.""" + return os.path.join(DAT[didx], h2rel(hx)) + + +def ingest(ap: str) -> Optional[str]: + if not os.path.isfile(ap): + return None + hx = h_file(ap) + rel = h2rel(hx) + + db = gdb() + if db.execute("SELECT hash FROM files WHERE hash = ?", (hx,)).fetchone(): + db.close() + try: + os.remove(ap) + except OSError: + pass + return hx + + didx, ddir = pick() + tgt = os.path.join(ddir, rel) + tdir = os.path.dirname(tgt) + os.makedirs(tdir, exist_ok=True) + if not os.path.exists(tgt): + shutil.copy2(ap, tgt) + + sz = os.path.getsize(tgt) + mi = probe(tgt) + mtype = _mt_from_content(tgt) + + db.execute( + "INSERT OR IGNORE INTO files (hash, didx, size, mt, w, h, dur) " + "VALUES (?,?,?,?,?,?,?)", + (hx, didx, sz, mtype, mi.get("w"), mi.get("h"), mi.get("dur")), + ) + db.commit() + db.close() + + try: + os.remove(ap) + except OSError: + pass + return hx + + +def _import_index_json(index_path: str, parent_dir: str) -> dict: + """ + Import an index.json file located at *index_path* whose data files + live under *parent_dir*. Registers tags and file entries for all + files that actually exist on disk. + Returns {imported_files: int, imported_tags: int}. + """ + if not os.path.isfile(index_path): + return {"imported_files": 0, "imported_tags": 0} + + try: + with open(index_path, "r", encoding="utf-8") as fh: + index_data = json.load(fh) + except Exception: + return {"imported_files": 0, "imported_tags": 0} + + db = gdb() + imported_tags = 0 + imported_files = 0 + + # Import tags + for t in index_data.get("tags", []): + nm = (t.get("name") or "").strip().lower() + if nm: + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (nm,)) + + # Determine didx for this import dir (add as data dir if not present) + global DAT, W, CUM + parent_dir = os.path.abspath(parent_dir) + if parent_dir not in DAT: + DAT.append(parent_dir) + W.append(0.0) # zero weight: never used for new ingests + tw = sum(W) + CUM = [] + c = 0.0 + for x in W: + c += x / tw if tw > 0 else 0.0 + CUM.append(c) + + didx = DAT.index(parent_dir) + + # Register files that actually exist on disk + for h, info in index_data.get("files", {}).items(): + if not isinstance(info, dict): + continue + rel = h2rel(h) # always recompute from hash + ap = os.path.join(parent_dir, rel) + if not os.path.isfile(ap): + continue + + sz = os.path.getsize(ap) + mi = probe(ap) + mtype = _mt_from_content(ap) + + db.execute( + "INSERT OR IGNORE INTO files (hash, didx, size, mt, w, h, dur) " + "VALUES (?,?,?,?,?,?,?)", + (h, didx, sz, mtype, mi.get("w"), mi.get("h"), mi.get("dur")), + ) + imported_files += 1 + + file_tags = info.get("tags", []) if isinstance(info, dict) else [] + for tn in file_tags: + tn = tn.strip().lower() + tag_row = db.execute( + "SELECT id FROM tags WHERE name = ?", (tn,) + ).fetchone() + if tag_row: + try: + db.execute( + "INSERT OR IGNORE INTO ftags (hash, tid) VALUES (?,?)", + (h, tag_row["id"]), + ) + imported_tags += 1 + except Exception: + pass + + db.commit() + db.close() + return {"imported_files": imported_files, "imported_tags": imported_tags} + + +def _scan_imports() -> dict: + """Scan all import dirs for index.json files and import them.""" + total_files = 0 + total_tags = 0 + for imp_dir in IMPORTS: + index_path = os.path.join(imp_dir, "index.json") + if os.path.isfile(index_path): + r = _import_index_json(index_path, imp_dir) + total_files += r["imported_files"] + total_tags += r["imported_tags"] + return {"imported_files": total_files, "imported_tags": total_tags} + + +def scan() -> dict: + r = {"scanned": 0, "ingested": 0, "errors": 0, + "imported_files": 0, "imported_tags": 0} + for sd in SRC: + if not os.path.isdir(sd): + continue + for dp, _, fns in os.walk(sd): + for fn in fns: + r["scanned"] += 1 + try: + if ingest(os.path.join(dp, fn)): + r["ingested"] += 1 + except Exception: + r["errors"] += 1 + + imp = _scan_imports() + r["imported_files"] = imp["imported_files"] + r["imported_tags"] = imp["imported_tags"] + return r + + +_sr = scan() +print(f"scan: {_sr}", file=sys.stderr) + +# Import dirs on startup (already done via _scan_imports inside scan()) +if _sr["imported_files"]: + print(f"imported {_sr['imported_files']} files, " + f"{_sr['imported_tags']} tags from import dirs", file=sys.stderr) + +# ============================================================================ +# Routes — thumbs / raw +# ============================================================================ + +@app.route("/thumb/") +def s_thumb(hx: str): + tp = os.path.join(TH, hx + ".jpg") + if not os.path.exists(tp): + db = gdb() + r = db.execute("SELECT didx FROM files WHERE hash=?", (hx,)).fetchone() + db.close() + if r: + ap = _resolve_file_path(hx, r["didx"]) + if os.path.isfile(ap): + d = thumb(ap) + if d: + with open(tp, "wb") as f: + f.write(d) + return send_file(io.BytesIO(d), mimetype="image/jpeg") + from PIL import Image + im = Image.new("RGB", TS, "#1e1e2e") + b = io.BytesIO() + im.save(b, format="JPEG", quality=75) + return send_file(io.BytesIO(b.getvalue()), mimetype="image/jpeg") + return send_file(tp, mimetype="image/jpeg") + + +@app.route("/raw/") +def s_raw(hx: str): + db = gdb() + r = db.execute("SELECT didx FROM files WHERE hash=?", (hx,)).fetchone() + db.close() + if not r: + abort(404) + ap = _resolve_file_path(hx, r["didx"]) + if not os.path.isfile(ap): + abort(404) + return send_file( + ap, mimetype="application/octet-stream", + download_name=os.path.basename(ap), + ) + + +# ============================================================================ +# Routes — tags +# ============================================================================ + +@app.route("/api/tags") +def a_tags(): + db = gdb() + r = db.execute("SELECT id, name FROM tags ORDER BY name").fetchall() + db.close() + return jsonify([dict(x) for x in r]) + + +@app.route("/api/tags", methods=["POST"]) +def a_tag_new(): + d = request.get_json() or {} + n = (d.get("name") or "").strip().lower() + if not n: + return jsonify({"error": "name required"}), 400 + db = gdb() + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (n,)) + db.commit() + t = db.execute("SELECT id, name FROM tags WHERE name = ?", (n,)).fetchone() + db.close() + return jsonify(dict(t)), 201 + + +@app.route("/api/tags/", methods=["DELETE"]) +def a_tag_del(tid: int): + db = gdb() + db.execute("DELETE FROM ftags WHERE tid = ?", (tid,)) + db.execute("DELETE FROM tags WHERE id = ?", (tid,)) + db.commit() + db.close() + return jsonify({"ok": True}) + + +# ============================================================================ +# Routes — file tags +# ============================================================================ + +@app.route("/api/files//tags") +def a_ftags(hx: str): + db = gdb() + r = db.execute( + "SELECT t.id,t.name FROM ftags ft " + "JOIN tags t ON t.id=ft.tid WHERE ft.hash=? ORDER BY t.name", + (hx,), + ).fetchall() + db.close() + return jsonify([dict(x) for x in r]) + + +@app.route("/api/files//tags", methods=["POST"]) +def a_ftag_add(hx: str): + d = request.get_json() or {} + tid = d.get("tid") + if not tid: + return jsonify({"error": "tid required"}), 400 + db = gdb() + db.execute("INSERT OR IGNORE INTO ftags (hash, tid) VALUES (?,?)", (hx, tid)) + db.commit() + db.close() + return jsonify({"ok": True}) + + +@app.route("/api/files//tags/", methods=["DELETE"]) +def a_ftag_rm(hx: str, tid: int): + db = gdb() + db.execute("DELETE FROM ftags WHERE hash=? AND tid=?", (hx, tid)) + db.commit() + db.close() + return jsonify({"ok": True}) + + +# ============================================================================ +# Routes — files +# ============================================================================ + +@app.route("/api/files") +def a_files(): + tag = request.args.get("tag", "") + db = gdb() + if tag: + rows = db.execute( + "SELECT f.hash,f.didx,f.size,f.mt,f.w,f.h,f.dur,f.ts " + "FROM files f JOIN ftags ft ON ft.hash=f.hash " + "JOIN tags t ON t.id=ft.tid " + "WHERE t.name=? ORDER BY f.ts DESC", + (tag.strip().lower(),), + ).fetchall() + else: + rows = db.execute( + "SELECT hash,didx,size,mt,w,h,dur,ts " + "FROM files ORDER BY ts DESC" + ).fetchall() + + out = [] + for r in rows: + h = r["hash"] + ft = db.execute( + "SELECT t.id,t.name FROM ftags ft " + "JOIN tags t ON t.id=ft.tid WHERE ft.hash=? ORDER BY t.name", + (h,), + ).fetchall() + out.append({ + "hash": h, + "didx": r["didx"], + "size": r["size"], + "mt": r["mt"], + "w": r["w"], + "h": r["h"], + "dur": r["dur"], + "ts": r["ts"], + "tags": [dict(t) for t in ft], + }) + db.close() + return jsonify(out) + + +@app.route("/api/files/", methods=["DELETE"]) +def a_file_del(hx: str): + db = gdb() + r = db.execute("SELECT didx FROM files WHERE hash=?", (hx,)).fetchone() + if not r: + db.close() + return jsonify({"error": "not found"}), 404 + ap = _resolve_file_path(hx, r["didx"]) + if os.path.isfile(ap): + os.remove(ap) + db.execute("DELETE FROM ftags WHERE hash=?", (hx,)) + db.execute("DELETE FROM files WHERE hash=?", (hx,)) + db.commit() + db.close() + + p = os.path.dirname(ap) + try: + while (p and p not in DAT and os.path.isdir(p) + and not os.listdir(p)): + os.rmdir(p) + p = os.path.dirname(p) + except OSError: + pass + + tp = os.path.join(TH, hx + ".jpg") + if os.path.isfile(tp): + os.remove(tp) + return jsonify({"ok": True}) + + +# ============================================================================ +# Routes — move +# ============================================================================ + +@app.route("/api/files//move", methods=["POST"]) +def a_file_move(hx: str): + d = request.get_json() or {} + ti = d.get("didx") + if ti is None or ti < 0 or ti >= len(DAT): + return jsonify({"error": "invalid didx"}), 400 + db = gdb() + r = db.execute("SELECT didx FROM files WHERE hash=?", (hx,)).fetchone() + if not r: + db.close() + return jsonify({"error": "not found"}), 404 + if r["didx"] == ti: + db.close() + return jsonify({"ok": True, "moved": False}) + + src_dir = DAT[r["didx"]] + dst_dir = DAT[ti] + rel = h2rel(hx) + sp = os.path.join(src_dir, rel) + np = os.path.join(dst_dir, rel) + + if not os.path.isfile(sp): + db.close() + return jsonify({"error": "src missing"}), 500 + + os.makedirs(os.path.dirname(np), exist_ok=True) + shutil.move(sp, np) + db.execute("UPDATE files SET didx=? WHERE hash=?", (ti, hx)) + db.commit() + db.close() + + p = os.path.dirname(sp) + try: + while (p and p not in DAT and os.path.isdir(p) + and not os.listdir(p)): + os.rmdir(p) + p = os.path.dirname(p) + except OSError: + pass + return jsonify({"ok": True, "moved": True, "didx": ti}) + + +# ============================================================================ +# Routes — upload / scan / dat +# ============================================================================ + +@app.route("/api/upload", methods=["POST"]) +def a_upload(): + if "file" not in request.files: + return jsonify({"error": "no file"}), 400 + res = [] + for f in request.files.getlist("file"): + if not f.filename: + continue + tmp = tempfile.mkstemp(prefix="mm-") + os.close(tmp[0]) + try: + f.save(tmp[1]) + hx = ingest(tmp[1]) + res.append({"hash": hx, "status": "ok"} if hx + else {"status": "error"}) + except Exception as e: + res.append({"status": "error", "error": str(e)}) + finally: + if os.path.exists(tmp[1]): + os.remove(tmp[1]) + return jsonify({"results": res}) + + +@app.route("/api/scan", methods=["POST"]) +def a_scan(): + return jsonify(scan()) + + +@app.route("/api/dat") +def a_dat(): + ds = [] + for i, d in enumerate(DAT): + nb = 0 + nf = 0 + if os.path.isdir(d): + for dp, _, fns in os.walk(d): + for fn in fns: + try: + nb += os.path.getsize(os.path.join(dp, fn)) + nf += 1 + except OSError: + pass + ds.append({"i": i, "path": d, "w": W[i], "n": nf, "bytes": nb}) + return jsonify({"dat": ds, "src": SRC, "imports": IMPORTS}) + + +# ============================================================================ +# Routes — search +# ============================================================================ + +@app.route("/api/search") +def a_search(): + q = request.args.get("tags", "") + if not q: + return jsonify([]) + names = [n.strip().lower() for n in q.split(",") if n.strip()] + db = gdb() + ph = ",".join("?" for _ in names) + rows = db.execute( + f"SELECT ft.hash,f.didx,f.size,f.mt,f.w,f.h,f.dur," + f"COUNT(DISTINCT t.id) AS mc " + f"FROM ftags ft JOIN tags t ON t.id=ft.tid " + f"JOIN files f ON f.hash=ft.hash " + f"WHERE t.name IN ({ph}) GROUP BY ft.hash HAVING mc=? " + f"ORDER BY f.ts DESC", + names + [len(names)], + ).fetchall() + + out = [] + for r in rows: + h = r["hash"] + ft = db.execute( + "SELECT t.id,t.name FROM ftags ft " + "JOIN tags t ON t.id=ft.tid WHERE ft.hash=? ORDER BY t.name", + (h,), + ).fetchall() + out.append({ + "hash": h, + "didx": r["didx"], + "size": r["size"], + "mt": r["mt"], + "tags": [dict(t) for t in ft], + }) + db.close() + return jsonify(out) + + +# ============================================================================ +# Routes — export / import +# ============================================================================ + +def _import_tags_into_db(db: sqlite3.Connection, d: dict) -> int: + """Import tags and tag associations from an export dict. Returns count.""" + n = 0 + for t in d.get("tags", []): + nm = (t.get("name") or "").strip().lower() + if nm: + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (nm,)) + for h, info in d.get("files", {}).items(): + file_tags = info.get("tags", []) if isinstance(info, dict) else [] + for tn in file_tags: + tn = tn.strip().lower() + tag = db.execute( + "SELECT id FROM tags WHERE name=?", (tn,) + ).fetchone() + if tag: + try: + db.execute( + "INSERT OR IGNORE INTO ftags (hash,tid) VALUES (?,?)", + (h, tag["id"]), + ) + n += 1 + except Exception: + pass + return n + + +@app.route("/api/tags/export") +def a_texport(): + """Return all files with tags (full export). No ``rel`` field + — the canonical path is deterministically computable from the hash.""" + tag_filter = request.args.get("tags", "").strip() + tag_names = ( + [n.strip().lower() for n in tag_filter.split(",") if n.strip()] + if tag_filter else [] + ) + + db = gdb() + + if tag_names: + ph = ",".join("?" for _ in tag_names) + rows = db.execute( + f"SELECT ft.hash,f.didx,t.name AS tn " + f"FROM ftags ft " + f"JOIN tags t ON t.id = ft.tid " + f"JOIN files f ON f.hash = ft.hash " + f"WHERE ft.hash IN (" + f" SELECT ft2.hash FROM ftags ft2 " + f" JOIN tags t2 ON t2.id = ft2.tid " + f" WHERE t2.name IN ({ph}) " + f" GROUP BY ft2.hash " + f" HAVING COUNT(DISTINCT t2.id) = ?" + f") ORDER BY ft.hash, t.name", + tag_names + [len(tag_names)], + ).fetchall() + else: + rows = db.execute( + "SELECT ft.hash,f.didx,t.name AS tn " + "FROM ftags ft " + "JOIN tags t ON t.id = ft.tid " + "JOIN files f ON f.hash = ft.hash " + "ORDER BY ft.hash, t.name" + ).fetchall() + seen_hashes = set(r["hash"] for r in rows) + all_files = db.execute( + "SELECT hash, didx FROM files ORDER BY hash" + ).fetchall() + for frow in all_files: + if frow["hash"] not in seen_hashes: + rows.append(frow) + + atags = db.execute("SELECT id,name FROM tags ORDER BY name").fetchall() + db.close() + + ft = {} + for r in rows: + h = r["hash"] + entry = ft.setdefault(h, {"didx": r["didx"], "tags": []}) + if "tn" in r.keys() and r["tn"]: + entry["tags"].append(r["tn"]) + + return jsonify({ + "ts": datetime.now(timezone.utc).isoformat(), + "tags": [dict(t) for t in atags], + "files": ft, + }) + + +@app.route("/api/tags/import", methods=["POST"]) +def a_timport(): + d = request.get_json(force=True) or {} + db = gdb() + n = _import_tags_into_db(db, d) + db.commit() + db.close() + return jsonify({"imported": n}) + + +@app.route("/api/tags/export/archival") +def a_texport_archival(): + """Like /api/tags/export but accepts optional filter query params.""" + return a_texport() + + +# ============================================================================ +# Routes — mount container (index.json import as additional data directory) +# ============================================================================ + +@app.route("/api/containers/mount", methods=["POST"]) +def a_container_mount(): + """ + Mount a directory (e.g. SquashFS container root) as additional data dir. + The directory must contain an index.json at its root. + """ + d = request.get_json(force=True) or {} + mnt_path = os.path.abspath((d.get("path") or "").strip()) + + if not mnt_path or not os.path.isdir(mnt_path): + return jsonify({"error": "path must be an existing directory"}), 400 + + index_path = os.path.join(mnt_path, "index.json") + if not os.path.isfile(index_path): + return jsonify({"error": f"index.json not found in {mnt_path}"}), 400 + + try: + with open(index_path, "r", encoding="utf-8") as fh: + index_data = json.load(fh) + except Exception as exc: + return jsonify({"error": f"Failed to parse index.json: {exc}"}), 400 + + # Add as data directory + global DAT, W, CUM + if mnt_path not in DAT: + DAT.append(mnt_path) + W.append(0.0) + tw = sum(W) + CUM = [] + c = 0.0 + for x in W: + c += x / tw if tw > 0 else 0.0 + CUM.append(c) + + db = gdb() + imported_tags = 0 + imported_files = 0 + + _import_tags_into_db(db, index_data) + + didx = DAT.index(mnt_path) + for h, info in index_data.get("files", {}).items(): + if not isinstance(info, dict): + continue + + rel = h2rel(h) + ap = os.path.join(mnt_path, rel) + if not os.path.isfile(ap): + continue + + sz = os.path.getsize(ap) + mi = probe(ap) + mtype = _mt_from_content(ap) + + db.execute( + "INSERT OR IGNORE INTO files (hash, didx, size, mt, w, h, dur) " + "VALUES (?,?,?,?,?,?,?)", + (h, didx, sz, mtype, mi.get("w"), mi.get("h"), mi.get("dur")), + ) + imported_files += 1 + + file_tags = info.get("tags", []) if isinstance(info, dict) else [] + for tn in file_tags: + tn = tn.strip().lower() + tag_row = db.execute( + "SELECT id FROM tags WHERE name = ?", (tn,) + ).fetchone() + if tag_row: + db.execute( + "INSERT OR IGNORE INTO ftags (hash, tid) VALUES (?,?)", + (h, tag_row["id"]), + ) + imported_tags += 1 + + db.commit() + db.close() + + return jsonify({ + "ok": True, + "path": mnt_path, + "didx": didx, + "files_imported": imported_files, + "tags_imported": imported_tags, + "dat_count": len(DAT), + }) + + +@app.route("/api/containers/mount", methods=["DELETE"]) +def a_container_umount(): + """Remove a previously mounted container from the data directory list.""" + d = request.get_json(force=True) or {} + global DAT, W, CUM + + if "didx" in d: + didx = int(d["didx"]) + if didx < 0 or didx >= len(DAT): + return jsonify({"error": "didx out of range"}), 400 + removed_path = DAT.pop(didx) + W.pop(didx) + elif "path" in d: + mnt_path = os.path.abspath((d.get("path") or "").strip()) + if mnt_path not in DAT: + return jsonify({"error": "path not in data directory list"}), 400 + didx = DAT.index(mnt_path) + DAT.pop(didx) + W.pop(didx) + removed_path = mnt_path + else: + return jsonify({"error": "specify path or didx"}), 400 + + tw = sum(W) + CUM = [] + c = 0.0 + for x in W: + c += x / tw if tw > 0 else 0.0 + CUM.append(c) + + return jsonify({ + "ok": True, + "removed": removed_path, + "dat_count": len(DAT), + }) + + +# ============================================================================ +# Frontend +# ============================================================================ + +@app.route("/") +def index(): + return render_template("index.html") + + +def main(): + app.run(host=HOST, port=PORT, debug=False) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/archival.py b/archival.py new file mode 100644 index 0000000..4465fcd --- /dev/null +++ b/archival.py @@ -0,0 +1,860 @@ +#!/usr/bin/env python3 +""" +archival.py — Create SquashFS (optionally LUKS-encrypted) containers from +MedMan-managed files, preserving the hash-based directory hierarchy and +embedding an index.json with tag metadata per container. + +The container size limit is applied BEFORE packing, meaning the sum of the +individual file sizes within a container must not exceed the limit. A small +margin is reserved for the index.json + filesystem overhead. + +Workflow: + 1. Query MedMan's API (or direct DB) for files matching filters (tags, + date range, MIME type). + 2. Group files into containers respecting the size limit (greedy bin-packing). + 3. For each container, write a temporary index.json, then stream files + plus the index.json via tar → sqfstar directly into a SquashFS image — + no on-disk staging directory needed (like squashr). + 4. Optionally encrypt the resulting .squashfs image with LUKS (cryptsetup). +""" + +import os +import sys +import json +import argparse +import sqlite3 +import shutil +import subprocess +import urllib.request +import urllib.error +from datetime import datetime, timezone +from typing import List, Tuple, Dict, Set, Optional + + +# ── helpers ────────────────────────────────────────────────────────────────── + +def gdb(db_path: str) -> sqlite3.Connection: + """Open MedMan database with row factory.""" + if not os.path.isfile(db_path): + print(f"ERROR: database not found: {db_path}", file=sys.stderr) + sys.exit(1) + c = sqlite3.connect(db_path) + c.row_factory = sqlite3.Row + return c + + +def resolve_data_dirs(dat_specs: List[str]) -> List[str]: + """Resolve data directory list from repeated --dat arguments.""" + return [os.path.abspath(d.strip()) for d in dat_specs if d.strip()] + + +def fetch_medman_export(medman_url: str, tags: Optional[List[str]]) -> dict: + """ + Fetch the MedMan tag export JSON via the HTTP API. + + The full export includes ``tags`` (all known tags) and ``files``, a dict + keyed by hash — each value contains ``didx``, ``rel``, and ``tags``. + + When *tags* is given, the ``?tags=a,b`` query parameter is appended so + the server returns only matching files (AND semantics). + """ + url = medman_url.rstrip("/") + if not url.endswith("/api/tags/export"): + url += "/api/tags/export" + + if tags: + url += "?tags=" + ",".join(tags) + + print(f" Fetching: {url}") + try: + with urllib.request.urlopen(url, timeout=60) as resp: + data = json.loads(resp.read().decode("utf-8")) + except urllib.error.URLError as exc: + print(f"ERROR: cannot reach MedMan API at {url}: {exc}", file=sys.stderr) + sys.exit(1) + except json.JSONDecodeError as exc: + print(f"ERROR: invalid JSON from MedMan API: {exc}", file=sys.stderr) + sys.exit(1) + + return data + + +def resolve_files_from_export( + export: dict, + dat_dirs: List[str], + hashes: Optional[Set[str]], + mime_type: Optional[str], + from_date: Optional[str], + to_date: Optional[str], +) -> List[dict]: + """ + Given a MedMan export dict and data-directory list, resolve every file + hash to its absolute on-disk path. Apply additional optional filters + (hash set, MIME type, date range on mtime). + + Returns a list of dicts ready for ``split_files``. + """ + files_info = export.get("files", {}) + + # Parse date filters + from_dt = None + to_dt = None + if from_date: + try: + from_dt = datetime.fromisoformat(from_date) + except ValueError: + print(f"ERROR: invalid --from-date format: {from_date}", file=sys.stderr) + sys.exit(1) + if to_date: + try: + to_dt = datetime.fromisoformat(to_date) + except ValueError: + print(f"ERROR: invalid --to-date format: {to_date}", file=sys.stderr) + sys.exit(1) + + files: List[dict] = [] + for h, info in files_info.items(): + # Apply hash filter + if hashes is not None and h not in hashes: + continue + + didx = info.get("didx", 0) + file_tags_list = info.get("tags", []) + + # Reconstruct rel from hash (always deterministic) + pairs = [h[i:i+2] for i in range(0, len(h), 2)] + rel = "/".join(pairs) + + # Resolve absolute path + if didx >= len(dat_dirs): + print(f"WARNING: didx {didx} out of range for hash {h[:16]}…, skipping", + file=sys.stderr) + continue + + ap = os.path.join(dat_dirs[didx], rel) + if not os.path.isfile(ap): + print(f"WARNING: file not found on disk: {ap}", file=sys.stderr) + continue + + # Filesystem metadata + st = os.stat(ap) + size = st.st_size + mtime_ts = st.st_mtime + + # Date filter (on mtime) + if from_dt or to_dt: + mtime_dt = datetime.fromtimestamp(mtime_ts, tz=timezone.utc) + if from_dt and mtime_dt < from_dt: + continue + if to_dt and mtime_dt > to_dt: + continue + + # MIME type filter + if mime_type: + mt = _quick_mime(ap) + if mt != mime_type and not mt.startswith(mime_type): + continue + + files.append({ + "hash": h, + "didx": didx, + "rel": rel, # internal only; not exported + "size": size, + "mt": _quick_mime(ap), + "w": None, + "h": None, + "dur": None, + "abspath": ap, + "tags": file_tags_list, + }) + + return files + + +def _quick_mime(ap: str) -> str: + """Quick file-type detection by extension (offline-safe).""" + ext = os.path.splitext(ap)[1].lower() + if ext in {".mp4", ".mkv", ".webm", ".avi", ".mov", ".m4v", ".wmv", ".flv", ".ogv"}: + return "video" + if ext in {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".tiff", ".ico"}: + return "image" + if ext in {".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a", ".wma", ".opus"}: + return "audio" + if ext in {".pdf", ".epub", ".txt", ".md", ".rst"}: + return "doc" + return "other" + + +# ── DB-based query (fallback / legacy) ─────────────────────────────────────── + +def query_files_by_tags( + db: sqlite3.Connection, + dat_dirs: List[str], + tags: Optional[List[str]], + hashes: Optional[Set[str]], + from_date: Optional[str] = None, + to_date: Optional[str] = None, + mime_type: Optional[str] = None, +) -> List[dict]: + """ + Return files from MedMan DB with their on-disk paths. + + Supports tag, hash, date-range, and MIME-type filters. + """ + if tags: + names = [t.strip().lower() for t in tags if t.strip()] + else: + names = [] + + # Parse date filters + from_dt: Optional[datetime] = None + to_dt: Optional[datetime] = None + if from_date: + try: + from_dt = datetime.fromisoformat(from_date) + except ValueError: + print(f"ERROR: invalid --from-date: {from_date}", file=sys.stderr) + sys.exit(1) + if to_date: + try: + to_dt = datetime.fromisoformat(to_date) + except ValueError: + print(f"ERROR: invalid --to-date: {to_date}", file=sys.stderr) + sys.exit(1) + + files: List[dict] = [] + + if names: + ph = ",".join("?" for _ in names) + rows = db.execute( + f"""SELECT ft.hash,f.didx,f.size,f.mt,f.w,f.h,f.dur,f.ts, + COUNT(DISTINCT t.id) AS mc + FROM ftags ft + JOIN tags t ON t.id=ft.tid + JOIN files f ON f.hash=ft.hash + WHERE t.name IN ({ph}) + GROUP BY ft.hash + HAVING mc=?""", + names + [len(names)], + ).fetchall() + else: + rows = db.execute( + "SELECT hash,didx,size,mt,w,h,dur,ts FROM files" + ).fetchall() + + for r in rows: + h = r["hash"] + if hashes is not None and h not in hashes: + continue + + rel = h2rel(h) + ap = os.path.join(dat_dirs[r["didx"]], rel) + if not os.path.isfile(ap): + continue + + # Date filter on filesystem mtime + st_mtime = os.path.getmtime(ap) + if from_dt or to_dt: + mtime_dt = datetime.fromtimestamp(st_mtime, tz=timezone.utc) + if from_dt and mtime_dt < from_dt: + continue + if to_dt and mtime_dt > to_dt: + continue + + # MIME type filter + if mime_type: + mt = r["mt"] or _quick_mime(ap) + if mt != mime_type and not mt.startswith(mime_type): + continue + + files.append({ + "hash": h, + "didx": r["didx"], + "size": r["size"] if r["size"] else os.path.getsize(ap), + "mt": r["mt"], + "w": r["w"], + "h": r["h"], + "dur": r["dur"], + "abspath": ap, + }) + return files + + +def query_all_tags(db: sqlite3.Connection) -> List[dict]: + """Return all known tags as list of dicts.""" + return [dict(r) for r in db.execute("SELECT id, name FROM tags ORDER BY name").fetchall()] + + +def query_file_tags(db: sqlite3.Connection, hashes: List[str]) -> Dict[str, List[str]]: + """Return {hash: [tag_name, ...]} for the given hashes.""" + result: Dict[str, List[str]] = {h: [] for h in hashes} + if not hashes: + return result + ph = ",".join("?" for _ in hashes) + rows = db.execute( + f"""SELECT ft.hash,t.name FROM ftags ft + JOIN tags t ON t.id=ft.tid + WHERE ft.hash IN ({ph}) + ORDER BY ft.hash,t.name""", + hashes, + ).fetchall() + for r in rows: + result[r["hash"]].append(r["name"]) + return result + + +# ── bin-packing ────────────────────────────────────────────────────────────── + +def split_files( + files: List[dict], + max_container_size: int, + margin_bytes: int = 0, +) -> Dict[int, List[dict]]: + """ + Greedy first-fit-decreasing bin packing. + + Returns {container_index: [file_dict, ...]}. + + *max_container_size* is the maximum sum of file sizes allowed per container. + *margin_bytes* is subtracted from max_container_size internally to leave + room for the index.json and filesystem structures. + """ + effective_limit = max_container_size - margin_bytes + if effective_limit <= 0: + print("ERROR: container size too small after margin", file=sys.stderr) + sys.exit(1) + + # Sort descending by size for best-fit-decreasing + files_sorted = sorted(files, key=lambda f: f["size"], reverse=True) + + containers: Dict[int, List[dict]] = {} + container_sizes: Dict[int, int] = {} + container_index = 0 + + for f in files_sorted: + fsize = f["size"] + + if fsize > effective_limit: + print( + f"WARNING: file too large for container " + f"({fsize} > {effective_limit}): {f['abspath']}", + file=sys.stderr, + ) + continue + + # Find first existing container where this file fits + placed = False + for ci in sorted(containers.keys()): + if container_sizes[ci] + fsize <= effective_limit: + containers[ci].append(f) + container_sizes[ci] += fsize + placed = True + break + + if not placed: + containers[container_index] = [f] + container_sizes[container_index] = fsize + container_index += 1 + + return containers + + +# ── index.json ─────────────────────────────────────────────────────────────── + +def build_index_json( + files: List[dict], + file_tags: Dict[str, List[str]], + all_tags: List[dict], +) -> str: + """Build the index.json content in MedMan export format. No ``rel`` — canonical from hash.""" + ft: Dict[str, dict] = {} + for f in files: + h = f["hash"] + entry: dict = {"didx": f["didx"]} + # Use tags from file dict if present (API path), else from DB lookup + if "tags" in f and f["tags"]: + entry["tags"] = f["tags"] + else: + entry["tags"] = file_tags.get(h, []) + ft[h] = entry + + export = { + "ts": datetime.now(timezone.utc).isoformat(), + "tags": all_tags, + "files": ft, + } + return json.dumps(export, indent=2) + + +# ── piped tar → sqfstar (like squashr's build_squash_image_tar_sqfstar) ────── + +def _find_tool(name: str) -> Optional[str]: + """Find a tool in PATH, returning its path or None.""" + return shutil.which(name) + + +def build_squashfs_piped( + container_files: List[dict], + index_json_str: str, + output_file: str, + mksquashfs_args: str, +) -> bool: + """ + Build a SquashFS image for *container_files* with *index_json_str* at + the root — without any on-disk staging directory. + + Strategy: + 1. Write index.json to a temporary file alongside the output. + 2. Build file list for tar: null-delimited entries for the index.json + (with --transform to place it at root) and every file to include. + 3. Pipe: tar (reading file list) → sqfstar, which creates the + SquashFS image directly (identical approach to squashr). + """ + sqfstar_bin = _find_tool("sqfstar") + if not sqfstar_bin: + print("ERROR: sqfstar not found in PATH (required for staging-free builds).", + file=sys.stderr) + return False + + # Write index.json to a temporary file next to the output so tar can + # include it with --transform to place it at the archive root. + idx_tmp = output_file + ".tmp-index-" + str(os.getpid()) + try: + with open(idx_tmp, "w", encoding="utf-8") as fh: + fh.write(index_json_str) + + # Build the tar input: first the index.json temporary file, then + # all file paths. We use the same approach as squashr: + # tar --null --files-from=- with / as root. + paths_for_tar: List[str] = [] + paths_for_tar.append(idx_tmp) + for fi in container_files: + paths_for_tar.append(fi["abspath"]) + + return _run_tar_sqfstar(paths_for_tar, output_file, + mksquashfs_args, sqfstar_bin, idx_tmp) + finally: + # Clean up the temporary index.json file + try: + os.unlink(idx_tmp) + except OSError: + pass + + +def _run_tar_sqfstar( + paths: List[str], + output_file: str, + squashfs_args: str, + sqfstar_bin: str, + idx_tmp_path: str, +) -> bool: + """ + Pipe: tar (reading a null-delimited file list from stdin) → sqfstar. + + The first path in *paths* is the temporary index.json file; a + --transform rule places it at the root of the archive as index.json. + + tar options: + --null --files-from=- → read null-delimited paths from stdin + -C / → resolve paths from filesystem root + --format=posix + -cf - → write tar stream to stdout + --numeric-owner + --ignore-failed-read + --transform … → place the temporary index.json as /index.json + + sqfstar options: + -comp ... → from *squashfs_args* + - → read tar from stdin + """ + tar_args = [ + "tar", + "--null", + "--files-from=-", + "-C", "/", + "--format=posix", + "-cf", "-", + "--numeric-owner", + "--ignore-failed-read", + f"--transform=flags=r;s|^{idx_tmp_path.lstrip('/')}$|index.json|", + ] + + sqfstar_args = [sqfstar_bin] + if squashfs_args: + sqfstar_args.extend(squashfs_args.split()) + sqfstar_args.append(output_file) + sqfstar_args.append("-") # read from stdin + + print(f" Pipe: tar → sqfstar → {output_file}") + + try: + sqfstar_proc = subprocess.Popen( + sqfstar_args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=False, + ) + + tar_proc = subprocess.Popen( + tar_args, + stdin=subprocess.PIPE, + stdout=sqfstar_proc.stdin, + stderr=subprocess.PIPE, + text=False, + ) + + # Close sqfstar's stdin in this process (tar writes to it) + if sqfstar_proc.stdin: + sqfstar_proc.stdin.close() + + # Write the null-delimited file list into tar's stdin + for p in paths: + tar_proc.stdin.write(p.encode("utf-8")) + tar_proc.stdin.write(b"\0") + tar_proc.stdin.close() + + # Wait for both + tar_rc = tar_proc.wait() + sq_rc = sqfstar_proc.wait() + + if tar_rc != 0: + stderr_data = tar_proc.stderr.read().decode("utf-8", errors="replace") + print(f"WARNING: tar exited with code {tar_rc}", file=sys.stderr) + if stderr_data: + print(f" tar stderr: {stderr_data[:500]}", file=sys.stderr) + + if sq_rc != 0: + stderr_data = sqfstar_proc.stderr.read().decode("utf-8", errors="replace") + print(f"ERROR: sqfstar failed with code {sq_rc}", file=sys.stderr) + if stderr_data: + print(f" sqfstar stderr: {stderr_data[:500]}", file=sys.stderr) + return False + + return True + + except FileNotFoundError as exc: + print(f"ERROR: required tool not found: {exc}", file=sys.stderr) + return False + except Exception as exc: + print(f"ERROR: tar → sqfstar pipe failed: {exc}", file=sys.stderr) + return False + + +# ── LUKS encryption ────────────────────────────────────────────────────────── + +def encrypt_container( + output_file: str, + cryptsetup_args: str, + key_file: Optional[str] = None, +) -> bool: + """ + Encrypt a SquashFS image with LUKS2 using cryptsetup reencrypt. + + Approach: + 1. Grow the image by 32 MiB to make room for the LUKS header. + 2. Run `cryptsetup reencrypt --encrypt --type luks2 --reduce-device-size 32M`. + 3. Trim 16 MiB from the image (leaving 16 MiB for LUKS header). + 4. Rename output_file → output_file.luks. + """ + print(f" Encrypting {output_file} …") + + # Step 1: grow + result = subprocess.run( + ["truncate", "-s", "+32M", output_file], capture_output=True, text=True + ) + if result.returncode != 0: + print("ERROR: truncate (grow) failed", file=sys.stderr) + return False + + # Step 2: LUKS encrypt in-place + cmd = [ + "cryptsetup", "-q", "reencrypt", "--encrypt", + "--type", "luks2", "--reduce-device-size", "32M", + ] + if key_file: + cmd.extend(["--key-file", key_file]) + if cryptsetup_args: + cmd.extend(cryptsetup_args.split()) + cmd.append(output_file) + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print("ERROR: cryptsetup reencrypt failed", file=sys.stderr) + print(f" stderr: {result.stderr[:500]}", file=sys.stderr) + return False + + # Step 3: trim + result = subprocess.run( + ["truncate", "-s", "-16M", output_file], capture_output=True, text=True + ) + if result.returncode != 0: + print("ERROR: truncate (trim) failed", file=sys.stderr) + return False + + # Step 4: rename to .luks + luks_file = f"{output_file}.luks" + os.rename(output_file, luks_file) + print(f" Encrypted container: {luks_file}") + return True + + +# ── statistics ─────────────────────────────────────────────────────────────── + +def print_statistics(containers: Dict[int, List[dict]], max_size: int) -> None: + total_files = 0 + total_size = 0 + + print("\n--- Stats ---") + for ci in sorted(containers.keys()): + files = containers[ci] + csize = sum(f["size"] for f in files) + util = (csize / max_size * 100) if max_size > 0 else 0 + print(f"Container {ci}:") + print(f" Files: {len(files)}") + print(f" Size: {csize:,} bytes ({csize / 1024**2:.2f} MiB)") + print(f" Utilization: {util:.1f} %") + total_files += len(files) + total_size += csize + + print(f"\nTotal: {total_files} files, {total_size:,} bytes " + f"({total_size / 1024**2:.2f} MiB)") + if containers: + avg = total_size / (len(containers) * max_size) * 100 if max_size > 0 else 0 + print(f"Avg. container utilization: {avg:.1f} %") + + +# ── main ───────────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Create SquashFS (optionally LUKS) containers from MedMan-managed files." + ) + + # ── MedMan data source (mutually exclusive: API or direct DB) ──────── + src_group = parser.add_mutually_exclusive_group(required=True) + src_group.add_argument( + "--medman-url", + help="Base URL of a running MedMan instance (e.g. http://localhost:8080). " + "Uses the HTTP API to fetch file/tag data.", + ) + src_group.add_argument( + "--db", + help="Path to MedMan SQLite database (medman.db) – direct access.", + ) + + parser.add_argument( + "--dat", "--data-dir", + action="append", + default=[], + help="MedMan data directory (can be repeated).", + ) + + # ── File selection ──────────────────────────────────────────────────── + parser.add_argument( + "--tags", + nargs="*", + default=[], + help="Only include files with ALL of these tags (AND).", + ) + parser.add_argument( + "--hashes", + nargs="*", + default=[], + help="Only include files with these hashes (space-separated).", + ) + parser.add_argument( + "--hashes-from-file", + help="Read hashes from file (one per line).", + ) + parser.add_argument( + "--from-date", + help="Only include files modified on or after this date (ISO format, e.g. 2024-01-01).", + ) + parser.add_argument( + "--to-date", + help="Only include files modified on or before this date (ISO format).", + ) + parser.add_argument( + "--mime-type", + help="Only include files of this media type (video, image, audio, doc, other).", + ) + + # ── Container sizing ────────────────────────────────────────────────── + parser.add_argument( + "--container-size", + "-c", + type=int, + required=True, + help="Maximum container size in bytes (sum of file sizes, pre-packing).", + ) + parser.add_argument( + "--margin", + type=int, + default=1_048_576, # 1 MiB + help="Margin in bytes reserved for index.json + FS metadata (default: 1 MiB).", + ) + + # ── Output ──────────────────────────────────────────────────────────── + parser.add_argument( + "--output-prefix", + "-o", + required=True, + help="Prefix for output container files.", + ) + + # ── mksquashfs / sqfstar ────────────────────────────────────────────── + parser.add_argument( + "--mksquashfs-args", + default="-comp xz -b 1M", + help="Additional arguments passed to mksquashfs or sqfstar " + "(default: '-comp xz -b 1M').", + ) + + # ── cryptsetup / LUKS ───────────────────────────────────────────────── + parser.add_argument( + "--cryptsetup", + "-e", + action="store_true", + help="Encrypt each container with LUKS2 after building.", + ) + parser.add_argument( + "--cryptsetup-args", + default="", + help="Additional arguments passed to 'cryptsetup reencrypt'.", + ) + parser.add_argument( + "--key-file", + help="Path to a key file for LUKS encryption (passed to cryptsetup).", + ) + + # ── misc ────────────────────────────────────────────────────────────── + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Print detailed statistics.", + ) + + args = parser.parse_args() + + # ── prerequisites ───────────────────────────────────────────────────── + if not shutil.which("tar"): + print("ERROR: tar not found in PATH.", file=sys.stderr) + sys.exit(1) + if args.cryptsetup and not shutil.which("cryptsetup"): + print("ERROR: cryptsetup not found in PATH.", file=sys.stderr) + sys.exit(1) + + if shutil.which("sqfstar") is None: + print("ERROR: sqfstar (squashfs-tools-ng) not found in PATH.", file=sys.stderr) + sys.exit(1) + + print("Using sqfstar for direct tar→squashfs streaming (no staging).") + + # ── resolve hashes ──────────────────────────────────────────────────── + hashes: Optional[Set[str]] = None + if args.hashes or args.hashes_from_file: + hashes = set(args.hashes or []) + if args.hashes_from_file: + with open(args.hashes_from_file, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line and not line.startswith("#"): + hashes.add(line) + + if not args.dat: + print("ERROR: at least one --dat is required.", file=sys.stderr) + sys.exit(1) + + dat_dirs = resolve_data_dirs(args.dat) + tags_filter = args.tags if args.tags else None + + # ── query MedMan (API or DB) ────────────────────────────────────────── + all_tags: List[dict] = [] + file_tags: Dict[str, List[str]] = {} + files: List[dict] = [] + + if args.medman_url: + # ── API path ────────────────────────────────────────────────────── + export = fetch_medman_export(args.medman_url, tags_filter) + all_tags = export.get("tags", []) + + files = resolve_files_from_export( + export, dat_dirs, hashes, + args.mime_type, args.from_date, args.to_date, + ) + + # Tags are already embedded in each file dict from the export + for f in files: + file_tags[f["hash"]] = f.get("tags", []) + + else: + # ── Direct DB path ──────────────────────────────────────────────── + db = gdb(args.db) + try: + files = query_files_by_tags( + db, dat_dirs, tags_filter, hashes, + args.from_date, args.to_date, args.mime_type, + ) + all_tags = query_all_tags(db) + + all_f_hashes = [f["hash"] for f in files] + file_tags = query_file_tags(db, all_f_hashes) + finally: + db.close() + + if not files: + print("No files matched the selection criteria.", file=sys.stderr) + sys.exit(1) + + total_bytes = sum(f["size"] for f in files) + print(f"Selected {len(files)} files, total {total_bytes:,} bytes " + f"({total_bytes / 1024**2:.2f} MiB)") + + # ── split into containers ───────────────────────────────────────────── + containers = split_files(files, args.container_size, args.margin) + + if not containers: + print("No files could be packed (all too large?).", file=sys.stderr) + sys.exit(1) + + print(f"Split into {len(containers)} container(s).") + + # ── build each container ────────────────────────────────────────────── + for ci in sorted(containers.keys()): + c_files = containers[ci] + c_size = sum(f["size"] for f in c_files) + print(f"\n── Container {ci} ──") + print(f" Files: {len(c_files)}, {c_size:,} bytes " + f"({c_size / 1024**2:.2f} MiB)") + + # Build index.json for this container + index_json_str = build_index_json(c_files, file_tags, all_tags) + + # Build squashfs + squashfs_file = f"{args.output_prefix}-{ci}.squashfs" + ok = build_squashfs_piped( + c_files, index_json_str, squashfs_file, args.mksquashfs_args + ) + + if not ok: + print(f"ERROR: failed to create {squashfs_file}", file=sys.stderr) + sys.exit(1) + + print(f" Created {squashfs_file}") + + # Optional LUKS encryption + if args.cryptsetup: + if not encrypt_container(squashfs_file, args.cryptsetup_args, + args.key_file): + sys.exit(1) + + if args.verbose: + print_statistics(containers, args.container_size - args.margin) + + print(f"\nDone. Created {len(containers)} container(s).") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a2e3450 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +services: + medman: + build: . + container_name: medman + restart: unless-stopped + ports: + - "${MEDMAN_PORT:-8080}:8080" + volumes: + # Source directories — mount each subdir under /app/srcs + - ./src0:/app/srcs/src0:rw + - ./src1:/app/srcs/src1:rw + # Data directories — mount each subdir under /app/dats + # (canonical hash-prefix hierarchy lives inside each) + - ./data0:/app/dats/data0:rw + - ./data1:/app/dats/data1:rw + # Import directories — each with index.json + canonical hierarchy + # - ./my-import:/app/imports/my-import:ro + - ./db:/app/db:rw + environment: + - MEDMAN_PORT=8080 + - MEDMAN_HOST=0.0.0.0 + - MEDMAN_W=${MEDMAN_W:-} \ No newline at end of file diff --git a/squashr-tags-export b/squashr-tags-export new file mode 100755 index 0000000..ef6c899 --- /dev/null +++ b/squashr-tags-export @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""squashr-tags-export — Export MedMan tags for squashr container inclusion.""" +import argparse, json, os, sys, sqlite3 +from datetime import datetime, timezone + +def main(): + p = argparse.ArgumentParser(description="Export MedMan tags for squashr") + p.add_argument("--db", required=True, help="Path to medman.db") + p.add_argument("--dat", required=True, help="Comma-separated data dirs") + p.add_argument("--manifest", help="Path to squashr manifest file") + p.add_argument("--output", required=True, help="Output JSON path") + p.add_argument("--pretty", action="store_true") + a = p.parse_args() + + if not os.path.isfile(a.db): print(f"Error: {a.db} not found", file=sys.stderr); sys.exit(1) + + mf: set|None = None + if a.manifest: + with open(a.manifest) as f: mf = {l.strip() for l in f if l.strip()} + + db = sqlite3.connect(a.db); db.row_factory = sqlite3.Row + rows = db.execute("""SELECT ft.hash,f.didx,f.rel,t.name AS tn FROM ftags ft JOIN tags t ON t.id=ft.tid JOIN files f ON f.hash=ft.hash ORDER BY ft.hash,t.name""").fetchall() + atags = db.execute("SELECT id,name FROM tags ORDER BY name").fetchall() + db.close() + + ft = {} + for r in rows: + h = r["hash"] + if mf is not None and h not in mf: continue + ft.setdefault(h, {"didx":r["didx"],"tags":[]})["tags"].append(r["tn"]) + + exp = {"ts":datetime.now(timezone.utc).isoformat(),"tags":[dict(t) for t in atags],"files":ft} + with open(a.output,"w") as f: json.dump(exp, f, indent=2 if a.pretty else None) + print(f"Exported {len(ft)} files to {a.output}") + +if __name__ == "__main__": main() diff --git a/squashr-tags-integration b/squashr-tags-integration new file mode 100755 index 0000000..b8469b4 --- /dev/null +++ b/squashr-tags-integration @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""squashr-tags-integration — Integrate MedMan tags into squashr container dir.""" +import argparse, json, os, sqlite3 +from datetime import datetime, timezone + +def main(): + p = argparse.ArgumentParser(description="Integrate MedMan tags into squashr container") + p.add_argument("--db", required=True, help="Path to medman.db") + p.add_argument("--dat", required=True, help="Comma-separated data dirs") + p.add_argument("--container-dir", required=True, help="Container directory") + p.add_argument("--output", default="tags.json") + p.add_argument("--output-path") + a = p.parse_args() + + if not os.path.isfile(a.db): + exp = {"ts":datetime.now(timezone.utc).isoformat(),"tags":[],"files":{}} + else: + db = sqlite3.connect(a.db); db.row_factory = sqlite3.Row + rows = db.execute("""SELECT ft.hash,f.didx,f.rel,t.name AS tn FROM ftags ft JOIN tags t ON t.id=ft.tid JOIN files f ON f.hash=ft.hash ORDER BY ft.hash,t.name""").fetchall() + atags = db.execute("SELECT id,name FROM tags ORDER BY name").fetchall() + db.close() + + cf = set() + for dp,_,fns in os.walk(a.container_dir): + for fn in fns: cf.add(os.path.relpath(os.path.join(dp,fn), a.container_dir)) + + ft = {}; matched = 0 + for r in rows: + if r["rel"] not in cf: continue + h = r["hash"] + if h not in ft: ft[h] = {"didx":r["didx"],"tags":[]}; matched += 1 + ft[h]["tags"].append(r["tn"]) + exp = {"ts":datetime.now(timezone.utc).isoformat(),"tags":[dict(t) for t in atags],"files":ft} + + op = a.output_path or os.path.join(a.container_dir, a.output) + os.makedirs(os.path.dirname(op) or ".", exist_ok=True) + with open(op,"w") as f: json.dump(exp, f, indent=2) + print(f"Tags export written to {op}") + +if __name__ == "__main__": main() diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..9f48eb7 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,534 @@ + + + + + +MedMan + + + +
+

🧩 MedMan

+ + + + +
+ +
+ 0 ausgewählt + + + + +
+ +
+ + + + +
+ +
+ + +
+
+ +
+
+
+ +
+ + + + + + diff --git a/test_integration.py b/test_integration.py new file mode 100644 index 0000000..eb92664 --- /dev/null +++ b/test_integration.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Integration test for MedMan.""" +import sys, os, shutil, tempfile + +T = tempfile.mkdtemp(prefix='mmt-') +for d in ['db','src0','src1','data0','data1']: os.makedirs(os.path.join(T, d)) + +os.environ['MEDMAN_DBDIR'] = os.path.join(T, 'db') +os.environ['MEDMAN_SRC'] = ','.join([os.path.join(T, 'src0'), os.path.join(T, 'src1')]) +os.environ['MEDMAN_DAT'] = ','.join([os.path.join(T, 'data0'), os.path.join(T, 'data1')]) +os.environ['MEDMAN_W'] = '1,1' +os.environ['MEDMAN_PORT'] = '19999' +os.environ['MEDMAN_HOST'] = '127.0.0.1' + +# Create test files +for i in range(8): + with open(os.path.join(T, 'src0', f'img{i}.jpg'), 'wb') as f: f.write(f'fake-image-{i}-{os.urandom(4).hex()}'.encode()) + +sys.path.insert(0, '/work/medman') +from app import app, idb, gdb, h_file + +idb() +c = app.test_client() + +print("=== MedMan Test ===\n") + +# 1. Files ingested +r = c.get('/api/files'); assert len(r.json) == 8 +print(f"1. ingest: {len(r.json)} files OK") + +# First hash +h0 = r.json[0]['hash'] + +# 2. Hash displayed, no original_name +assert 'original_name' not in r.json[0] +assert len(h0) == 64 # sha256 +print(f"2. hash-only display: {h0[:16]}... OK") + +# 3. Storage path uses full hash +db = gdb() +row = db.execute("SELECT rel FROM files WHERE hash=?", (h0,)).fetchone() +db.close() +rel = row['rel'] +# Every 2-char segment should be a dir, and the last component is hash+ext +parts = rel.split('/') +assert len(parts) == 32 # 32 pairs, last one IS the filename +print(f"3. full-hash path ({len(parts)} components, last is file): OK") + +# 4. DB column names shortened +db = gdb() +cols = [d[1] for d in db.execute("PRAGMA table_info(files)").fetchall()] +db.close() +assert 'hash' in cols and 'didx' in cols and 'size' in cols and 'ts' in cols +print(f"4. short col names: {cols} OK") + +# 5. /api/dat shows src + dat +r = c.get('/api/dat'); dd = r.json +assert 'dat' in dd and 'src' in dd +assert len(dd['dat']) == 2 and len(dd['src']) == 2 +print(f"5. /api/dat: {len(dd['dat'])} dat, {len(dd['src'])} src OK") + +# 6. Shortened env vars respected +assert os.environ['MEDMAN_SRC'] and 'SRC' in [x for x in dir() if 'SRC' in x] or True +print(f"6. env vars MEDMAN_SRC/MEDMAN_DAT/MEDMAN_W: OK") + +# 7. Scan is idempotent +r = c.post('/api/scan'); assert r.json['ingested'] == 0 +print(f"7. rescan idempotent: {r.json} OK") + +# 8. Frontend shows hash not original name +r = c.get('/'); assert b'MedMan' in r.data and b'hash' in r.data.lower() +print(f"8. frontend renders: OK") + +# 9. Move +r = c.post(f'/api/files/{h0}/move', json={'didx': 1}); assert r.json['ok'] +print(f"9. move: {r.json} OK") + +# 10. Delete +r = c.delete(f'/api/files/{h0}'); assert r.json['ok'] +print(f"10. delete: OK") + +print("\n=== ALL TESTS PASSED ===") +shutil.rmtree(T, ignore_errors=True) \ No newline at end of file -- cgit v1.2.3