diff options
| -rw-r--r-- | .env.example | 2 | ||||
| -rw-r--r-- | .gitignore | 110 | ||||
| -rw-r--r-- | Dockerfile | 18 | ||||
| -rw-r--r-- | README.md | 236 | ||||
| -rwxr-xr-x | app.py | 1019 | ||||
| -rw-r--r-- | archival.py | 860 | ||||
| -rw-r--r-- | docker-compose.yml | 22 | ||||
| -rwxr-xr-x | squashr-tags-export | 36 | ||||
| -rwxr-xr-x | squashr-tags-integration | 40 | ||||
| -rw-r--r-- | templates/index.html | 534 | ||||
| -rw-r--r-- | test_integration.py | 83 |
11 files changed, 2960 insertions, 0 deletions
diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..41c29f0 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +MEDMAN_PORT=8080 +MEDMAN_W=
\ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0bb0d5e --- /dev/null +++ b/.gitignore @@ -0,0 +1,110 @@ +*.pyc +__pycache__/ +*.egg-info/ +.venv/ +.env +data0 +data1 +src0 +src1 +db + +# Created by https://www.toptal.com/developers/gitignore/api/vim,linux,windows,macos +# Edit at https://www.toptal.com/developers/gitignore?templates=vim,linux,windows,macos + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/vim,linux,windows,macos + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7963438 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.11-slim-bookworm + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + squashfs-tools \ + cryptsetup-bin \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir flask pillow jinja2 + +WORKDIR /app +COPY . . + +# Base directories (mount subdirs via docker-compose) +RUN mkdir -p /app/db /app/srcs /app/dats /app/imports + +EXPOSE 8080 +CMD ["python", "app.py"]
\ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..df1f37e --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +# MedMan — Self-Hosted Media Manager with Tagging & squashr Integration + +MedMan is a web-based media manager designed to work alongside +[squashr](https://git.kug.is/squashr.git) (SquashFS/OverlayFS/LUKS backup tool). + +## Overview + +``` +1. Drop files into source directories (or upload via web UI) + ↓ +2. MedMan scans/ingests → files are distributed into data directories + using a full-hash directory hierarchy (every 2 hex chars = one level) + ↓ +3. Browse & tag files through the web interface + ↓ +4. Export tags → archival.py packs them into SquashFS containers + ↓ +5. Burn containers to CD/DVD/Blu-Ray or archive offline + ↓ +6. Offline use: mount container, start MedMan, import index.json → + browse & search media with full tag metadata +``` + +## Configuration + +MedMan is configured primarily through **Docker volume mounts**. Environment +variables are minimal: + +| Variable | Description | Default | +|---|---|---| +| `MEDMAN_PORT` | HTTP port | `8080` | +| `MEDMAN_HOST` | Bind address | `0.0.0.0` | +| `MEDMAN_W` | Comma-separated weights for data-dir distribution | equal weight | + +### Directory Structure + +All directories are inside `/app` in the container and wired via +`docker-compose.yml` mounts: + +``` +/app/ +├── srcs/ ← source directories (immediate subdirs) +│ ├── src0/ ← mount: ./src0:/app/srcs/src0 +│ └── src1/ ← mount: ./src1:/app/srcs/src1 +├── dats/ ← data directories (immediate subdirs) +│ ├── data0/ ← mount: ./data0:/app/dats/data0 +│ └── data1/ ← mount: ./data1:/app/dats/data1 +├── imports/ ← auto-import directories (immediate subdirs, read-only) +│ └── my-archive/ ← mount: /mnt/archive:/app/imports/my-archive:ro +└── db/ ← SQLite database + thumbnails (auto-created) +``` + +**Key principle**: Every immediate subdirectory of `srcs/`, `dats/`, and +`imports/` is automatically discovered. No comma-separated path lists in +environment variables. + +### Data Distribution Weights + +`MEDMAN_W` controls how new files are distributed across data directories: + +```bash +# 3 data dirs — dir 0 gets 50%, dirs 1 and 2 get 25% each +MEDMAN_W=2,1,1 +``` + +If not set, distribution is equal across all data directories. + +## Hash-Based Storage + +Every file is SHA-256 hashed. All 32 byte-pairs of the 64-character hex +hash become directory levels: + +``` +Hash: a1b2c3d4e5f6...a1b2c3d4e5f6 (64 chars) +Path: data0/a1/b2/c3/d4/e5/f6/.../a1/b2/c3/d4/e5/f6 +``` + +- 32 directory levels + filename (last hex pair) +- Automatic deduplication via content hashing +- Distribution across data directories according to `MEDMAN_W` weights +- No `rel` column stored in the database — the path is always deterministically + computed from the hash + +## Quickstart (Docker) + +```bash +cd medman +docker compose up -d +``` + +MedMan is available at `http://localhost:8080`. + +Drop files into `src0/` or `src1/` — they are automatically scanned and +distributed into the data directories on startup. Upload is also supported +via the web UI. + +## Auto-Import on Startup + +Directories mounted under `/app/imports/` are scanned on startup (and on +each manual scan). Each subdirectory may contain: + +- `index.json` — a MedMan export file with tags and file references +- Files in the canonical hash-prefix hierarchy (same layout as data dirs) + +All tags and file registrations from `index.json` are imported automatically +if the referenced files exist on disk. Import directories receive zero +distribution weight (new files are never placed there). + +Example: Mount a read-only SquashFS container containing an `index.json` and +canonical file paths: + +```yaml +# docker-compose.yml +volumes: + - /mnt/my-archive:/app/imports/my-archive:ro +``` + +## Features + +- **Source directories**: files auto-ingested on startup, upload via UI +- **Manual scan button**: re-scan source and import directories anytime +- **Hash-based storage**: content-addressable, no duplicates +- **Multiple data directories**: weighted random distribution +- **Move**: relocate individual files or selections between data dirs +- **Tag system**: plain labels, no color coding +- **Tag search**: combined AND search across multiple tags +- **Bulk operations**: select-all, batch tag/untag/move/delete +- **Previous/Next navigation**: arrow keys or buttons in detail view +- **Checkbox selection**: click the checkbox on any thumbnail to toggle +- **Export/Import**: tags as JSON (export/import) +- **archival.py**: create SquashFS + LUKS containers directly from MedMan +- **Container mount API**: mount SquashFS containers as additional data dirs + +## Database Schema + +```sql +tags(id, name, ts) +files(hash PK, didx, size, mt, w, h, dur, ts) +ftags(id, hash FK→files, tid FK→tags, UNIQUE(hash, tid)) +``` + +`didx` is the index into the `dats/` subdirectory list. The on-disk path +is always `dats/<name>/<h2rel(hash)>`. + +## API + +| Endpoint | Method | Description | +|---|---|---| +| `/api/files` | GET | All files (`?tag=` filter) | +| `/api/files/<hash>` | DELETE | Delete file | +| `/api/files/<hash>/move` | POST | `{"didx":N}` move to data dir | +| `/api/files/<hash>/tags` | GET/POST | Get or add tags | +| `/api/files/<hash>/tags/<id>` | DELETE | Remove tag from file | +| `/api/tags` | GET/POST | List or create tags | +| `/api/tags/<id>` | DELETE | Delete tag | +| `/api/tags/export` | GET | Full tag export (`?tags=a,b` AND filter) | +| `/api/tags/import` | POST | Import tag JSON | +| `/api/search?tags=a,b` | GET | AND tag search | +| `/api/upload` | POST | Upload file(s) | +| `/api/scan` | POST | Scan sources + imports | +| `/api/dat` | GET | Data directory stats | +| `/api/containers/mount` | POST | Mount container dir as data dir | +| `/api/containers/mount` | DELETE | Unmount container dir | +| `/thumb/<hash>` | GET | JPEG thumbnail | +| `/raw/<hash>` | GET | Original file | + +## Export Format (index.json) + +```json +{ + "ts": "2025-01-15T12:00:00+00:00", + "tags": [{"id": 1, "name": "example"}], + "files": { + "a1b2c3d4...": {"didx": 0, "tags": ["example"]} + } +} +``` + +The on-disk path for each file is deterministic: `h2rel(hash)` → 32 hex-pair +directory levels. No `rel` field is needed. + +## archival.py — Creating SquashFS Containers + +```bash +# Build containers via MedMan API +python3 archival.py --medman-url http://localhost:8080 \ + --dat /app/data0 /app/data1 \ + --tags vacation 2024 \ + --container-size 4700372992 \ + --output-prefix ./archive + +# Build with LUKS encryption +python3 archival.py --db /app/db/medman.db \ + --dat /app/data0 /app/data1 \ + --tags photos \ + --container-size 25000000000 \ + --output-prefix ./photos \ + --cryptsetup --key-file /path/to/keyfile + +# Direct DB access (no running MedMan needed) +python3 archival.py --db /app/db/medman.db \ + --dat /app/data0 /app/data1 \ + --container-size 10000000000 \ + --output-prefix ./offline +``` + +The script uses `tar → sqfstar` piping (no staging directories, no temp +files beyond a single index.json) — exactly like squashr's +`build_squash_image_tar_sqfstar`. + +### Container Size Margin + +A margin of 1 MiB is reserved per container for the `index.json` and +filesystem metadata overhead. Adjust with `--margin` if needed. + +### Filter Options + +| Option | Description | +|---|---| +| `--tags a b` | Only files with ALL given tags (AND) | +| `--hashes a1b2...` | Specific hashes | +| `--hashes-from-file` | Read hashes from file | +| `--from-date 2024-01-01` | Modified on or after | +| `--to-date 2024-12-31` | Modified on or before | +| `--mime-type video` | Filter by media type | + +## squashr Integration + +MedMan containers produced by `archival.py` are standard SquashFS images +with an `index.json` at the root plus the canonical hash hierarchy. They +can be mounted and imported into any MedMan instance via the auto-import +feature or the container mount API. + +## License + +GNU AGPLv3 @@ -0,0 +1,1019 @@ +#!/usr/bin/env python3 +""" +MedMan — Self-hosted media manager with tagging & squashr integration. + +Source directories: files are ingested on startup / manual scan trigger. + All immediate subdirectories of /app/srcs are treated as source dirs. + +Data directories: files stored under full content-hash prefix paths + (2 hex chars = 1 byte per directory level, entire hash used). + All immediate subdirectories of /app/dats are treated as data dirs. + +Import directories: mounted under /app/imports. Each subdirectory may + contain an index.json at its root (MedMan export format) and files in + the canonical hash-prefix hierarchy. Tags and file registrations are + imported automatically on startup and re-checked on manual scan. + +Tag system (no color coding). Upload via web UI. Move files between data dirs. +""" + +import os, sys, io, json, hashlib, sqlite3, subprocess +import shutil, tempfile, secrets +from datetime import datetime, timezone +from typing import Optional + +from flask import Flask, render_template, request, jsonify, send_file, abort +from werkzeug.utils import secure_filename + +# ============================================================================ +# Configuration +# ============================================================================ + +PORT = int(os.environ.get("MEDMAN_PORT", "8080")) +HOST = os.environ.get("MEDMAN_HOST", "0.0.0.0") +DBDIR = os.path.join(os.getcwd(), "db") + +# Source directories: all immediate subdirs of /app/srcs +_SRC_ROOT = os.path.join(os.getcwd(), "srcs") +SRC: list[str] = [] +if os.path.isdir(_SRC_ROOT): + SRC = sorted([ + os.path.join(_SRC_ROOT, d) for d in os.listdir(_SRC_ROOT) + if os.path.isdir(os.path.join(_SRC_ROOT, d)) + ]) +if not SRC: + SRC = [os.path.join(os.getcwd(), "src0"), os.path.join(os.getcwd(), "src1")] + +# Data directories: all immediate subdirs of /app/dats +_DAT_ROOT = os.path.join(os.getcwd(), "dats") +DAT: list[str] = [] +if os.path.isdir(_DAT_ROOT): + DAT = sorted([ + os.path.join(_DAT_ROOT, d) for d in os.listdir(_DAT_ROOT) + if os.path.isdir(os.path.join(_DAT_ROOT, d)) + ]) +if not DAT: + DAT = [os.path.join(os.getcwd(), "data0"), os.path.join(os.getcwd(), "data1")] + +# Import directories: all immediate subdirs of /app/imports +# Each may contain an index.json + canonical hash-prefix hierarchy +_IMPORT_ROOT = os.path.join(os.getcwd(), "imports") +IMPORTS: list[str] = [] +if os.path.isdir(_IMPORT_ROOT): + IMPORTS = sorted([ + os.path.join(_IMPORT_ROOT, d) for d in os.listdir(_IMPORT_ROOT) + if os.path.isdir(os.path.join(_IMPORT_ROOT, d)) + ]) + +# Weights (comma-separated floats; default equal) +_w = os.environ.get("MEDMAN_W", "") +W: list[float] = [] +if _w.strip(): + W = [float(x.strip()) for x in _w.split(",") if x.strip()] +if not W: + W = [1.0] * len(DAT) +if len(W) != len(DAT): + raise RuntimeError("MEDMAN_W must have same count as data dirs") +tw = sum(W) +W = [x / tw for x in W] +CUM: list[float] = [] +c = 0.0 +for x in W: + c += x + CUM.append(c) + +DB = os.path.join(DBDIR, "medman.db") +TH = os.path.join(DBDIR, "thumbs") + +for d in [DBDIR, TH] + DAT + SRC + IMPORTS: + os.makedirs(d, exist_ok=True) + +app = Flask(__name__) + +# ============================================================================ +# Hash helpers +# ============================================================================ + +def h_file(ap: str) -> str: + h = hashlib.sha256() + with open(ap, "rb") as f: + while True: + b = f.read(65536) + if not b: + break + h.update(b) + return h.hexdigest() + + +def h2rel(hx: str) -> str: + """Every 2 hex chars → one directory level (32 levels + filename).""" + pairs = [hx[i:i + 2] for i in range(0, len(hx), 2)] + return "/".join(pairs) + + +def rel_for_disk(hx: str, didx: int) -> str: + """Full on-disk relative path for a given hash and data dir index.""" + return h2rel(hx) + + +def pick() -> tuple[int, str]: + r = secrets.randbelow(2**31) / (2**31) + for i, cu in enumerate(CUM): + if r <= cu: + return i, DAT[i] + return len(DAT) - 1, DAT[-1] + + +# ============================================================================ +# Database +# ============================================================================ + +def gdb(): + c = sqlite3.connect(DB) + c.row_factory = sqlite3.Row + c.execute("PRAGMA journal_mode=WAL") + c.execute("PRAGMA foreign_keys=ON") + return c + + +def idb(): + db = gdb() + db.executescript(""" + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + ts TEXT DEFAULT (datetime('now')) + ); + + -- rel removed: it is always deterministically h2rel(hash) + CREATE TABLE IF NOT EXISTS files ( + hash TEXT PRIMARY KEY, + didx INTEGER NOT NULL, + size INTEGER, + mt TEXT, + w INTEGER, + h INTEGER, + dur REAL, + ts TEXT DEFAULT (datetime('now')) + ); + + CREATE TABLE IF NOT EXISTS ftags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT NOT NULL, + tid INTEGER NOT NULL, + FOREIGN KEY (hash) REFERENCES files(hash) ON DELETE CASCADE, + FOREIGN KEY (tid) REFERENCES tags(id) ON DELETE CASCADE, + UNIQUE(hash, tid) + ); + CREATE INDEX IF NOT EXISTS ix_ft_h ON ftags(hash); + CREATE INDEX IF NOT EXISTS ix_ft_t ON ftags(tid); + CREATE INDEX IF NOT EXISTS ix_ta_n ON tags(name); + """) + # Drop 'rel' column if it still exists (migration from older schema) + cols = [r[1] for r in db.execute("PRAGMA table_info(files)").fetchall()] + if "rel" in cols: + db.execute("ALTER TABLE files DROP COLUMN rel") + db.commit() + db.close() + + +idb() + +# ============================================================================ +# Helpers +# ============================================================================ + +VE = {".mp4", ".mkv", ".webm", ".avi", ".mov", ".m4v", ".wmv", ".flv", ".ogv"} +IE = {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".tiff", ".ico"} +AE = {".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a", ".wma", ".opus"} +DE = {".pdf", ".epub", ".txt", ".md", ".rst"} +TS = (320, 240) + + +def _mt_from_content(ap: str) -> str: + """Determine media type from file content (magic bytes / ffprobe).""" + try: + r = subprocess.run( + ["file", "-b", "--mime-type", ap], + capture_output=True, text=True, timeout=5, + ) + mt = r.stdout.strip() + if mt.startswith("video/"): return "video" + if mt.startswith("image/"): return "image" + if mt.startswith("audio/"): return "audio" + if mt.startswith("application/pdf"): return "doc" + if mt.startswith("text/"): return "doc" + except Exception: + pass + try: + r = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "format=format_name", + "-of", "csv=p=0", ap], + capture_output=True, text=True, timeout=10, + ) + fmts = r.stdout.strip().lower() + if any(x in fmts for x in ("mp4", "mov", "matroska", "avi")): return "video" + if any(x in fmts for x in ("mp3", "aac", "flac", "wav", "ogg")): return "audio" + except Exception: + pass + try: + from PIL import Image + Image.open(ap) + return "image" + except Exception: + pass + return "other" + + +def thumb(ap: str) -> Optional[bytes]: + """Generate thumbnail.""" + try: + from PIL import Image + im = Image.open(ap) + im.thumbnail(TS) + b = io.BytesIO() + im.save(b, format="JPEG", quality=75) + return b.getvalue() + except Exception: + pass + try: + r = subprocess.run([ + "ffmpeg", "-y", "-loglevel", "error", "-i", ap, + "-ss", "5", "-vframes", "1", + "-vf", f"scale={TS[0]}:{TS[1]}:force_original_aspect_ratio=decrease," + f"pad={TS[0]}:{TS[1]}:(ow-iw)/2:(oh-ih)/2", + "-f", "image2", "pipe:1", + ], capture_output=True, timeout=30) + if r.returncode == 0 and r.stdout: + return r.stdout + except Exception: + pass + return None + + +def probe(ap: str) -> dict: + try: + r = subprocess.run([ + "ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_entries", "stream=width,height,duration", "-of", "json", ap, + ], capture_output=True, timeout=15) + if r.returncode == 0: + d = json.loads(r.stdout) + if d.get("streams"): + s = d["streams"][0] + return {"w": s.get("width"), "h": s.get("height"), + "dur": s.get("duration")} + except Exception: + pass + return {} + + +def _resolve_file_path(hx: str, didx: int) -> str: + """Given a hash and data-dir index, compute the absolute file path.""" + return os.path.join(DAT[didx], h2rel(hx)) + + +def ingest(ap: str) -> Optional[str]: + if not os.path.isfile(ap): + return None + hx = h_file(ap) + rel = h2rel(hx) + + db = gdb() + if db.execute("SELECT hash FROM files WHERE hash = ?", (hx,)).fetchone(): + db.close() + try: + os.remove(ap) + except OSError: + pass + return hx + + didx, ddir = pick() + tgt = os.path.join(ddir, rel) + tdir = os.path.dirname(tgt) + os.makedirs(tdir, exist_ok=True) + if not os.path.exists(tgt): + shutil.copy2(ap, tgt) + + sz = os.path.getsize(tgt) + mi = probe(tgt) + mtype = _mt_from_content(tgt) + + db.execute( + "INSERT OR IGNORE INTO files (hash, didx, size, mt, w, h, dur) " + "VALUES (?,?,?,?,?,?,?)", + (hx, didx, sz, mtype, mi.get("w"), mi.get("h"), mi.get("dur")), + ) + db.commit() + db.close() + + try: + os.remove(ap) + except OSError: + pass + return hx + + +def _import_index_json(index_path: str, parent_dir: str) -> dict: + """ + Import an index.json file located at *index_path* whose data files + live under *parent_dir*. Registers tags and file entries for all + files that actually exist on disk. + Returns {imported_files: int, imported_tags: int}. + """ + if not os.path.isfile(index_path): + return {"imported_files": 0, "imported_tags": 0} + + try: + with open(index_path, "r", encoding="utf-8") as fh: + index_data = json.load(fh) + except Exception: + return {"imported_files": 0, "imported_tags": 0} + + db = gdb() + imported_tags = 0 + imported_files = 0 + + # Import tags + for t in index_data.get("tags", []): + nm = (t.get("name") or "").strip().lower() + if nm: + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (nm,)) + + # Determine didx for this import dir (add as data dir if not present) + global DAT, W, CUM + parent_dir = os.path.abspath(parent_dir) + if parent_dir not in DAT: + DAT.append(parent_dir) + W.append(0.0) # zero weight: never used for new ingests + tw = sum(W) + CUM = [] + c = 0.0 + for x in W: + c += x / tw if tw > 0 else 0.0 + CUM.append(c) + + didx = DAT.index(parent_dir) + + # Register files that actually exist on disk + for h, info in index_data.get("files", {}).items(): + if not isinstance(info, dict): + continue + rel = h2rel(h) # always recompute from hash + ap = os.path.join(parent_dir, rel) + if not os.path.isfile(ap): + continue + + sz = os.path.getsize(ap) + mi = probe(ap) + mtype = _mt_from_content(ap) + + db.execute( + "INSERT OR IGNORE INTO files (hash, didx, size, mt, w, h, dur) " + "VALUES (?,?,?,?,?,?,?)", + (h, didx, sz, mtype, mi.get("w"), mi.get("h"), mi.get("dur")), + ) + imported_files += 1 + + file_tags = info.get("tags", []) if isinstance(info, dict) else [] + for tn in file_tags: + tn = tn.strip().lower() + tag_row = db.execute( + "SELECT id FROM tags WHERE name = ?", (tn,) + ).fetchone() + if tag_row: + try: + db.execute( + "INSERT OR IGNORE INTO ftags (hash, tid) VALUES (?,?)", + (h, tag_row["id"]), + ) + imported_tags += 1 + except Exception: + pass + + db.commit() + db.close() + return {"imported_files": imported_files, "imported_tags": imported_tags} + + +def _scan_imports() -> dict: + """Scan all import dirs for index.json files and import them.""" + total_files = 0 + total_tags = 0 + for imp_dir in IMPORTS: + index_path = os.path.join(imp_dir, "index.json") + if os.path.isfile(index_path): + r = _import_index_json(index_path, imp_dir) + total_files += r["imported_files"] + total_tags += r["imported_tags"] + return {"imported_files": total_files, "imported_tags": total_tags} + + +def scan() -> dict: + r = {"scanned": 0, "ingested": 0, "errors": 0, + "imported_files": 0, "imported_tags": 0} + for sd in SRC: + if not os.path.isdir(sd): + continue + for dp, _, fns in os.walk(sd): + for fn in fns: + r["scanned"] += 1 + try: + if ingest(os.path.join(dp, fn)): + r["ingested"] += 1 + except Exception: + r["errors"] += 1 + + imp = _scan_imports() + r["imported_files"] = imp["imported_files"] + r["imported_tags"] = imp["imported_tags"] + return r + + +_sr = scan() +print(f"scan: {_sr}", file=sys.stderr) + +# Import dirs on startup (already done via _scan_imports inside scan()) +if _sr["imported_files"]: + print(f"imported {_sr['imported_files']} files, " + f"{_sr['imported_tags']} tags from import dirs", file=sys.stderr) + +# ============================================================================ +# Routes — thumbs / raw +# ============================================================================ + +@app.route("/thumb/<hx>") +def s_thumb(hx: str): + tp = os.path.join(TH, hx + ".jpg") + if not os.path.exists(tp): + db = gdb() + r = db.execute("SELECT didx FROM files WHERE hash=?", (hx,)).fetchone() + db.close() + if r: + ap = _resolve_file_path(hx, r["didx"]) + if os.path.isfile(ap): + d = thumb(ap) + if d: + with open(tp, "wb") as f: + f.write(d) + return send_file(io.BytesIO(d), mimetype="image/jpeg") + from PIL import Image + im = Image.new("RGB", TS, "#1e1e2e") + b = io.BytesIO() + im.save(b, format="JPEG", quality=75) + return send_file(io.BytesIO(b.getvalue()), mimetype="image/jpeg") + return send_file(tp, mimetype="image/jpeg") + + +@app.route("/raw/<hx>") +def s_raw(hx: str): + db = gdb() + r = db.execute("SELECT didx FROM files WHERE hash=?", (hx,)).fetchone() + db.close() + if not r: + abort(404) + ap = _resolve_file_path(hx, r["didx"]) + if not os.path.isfile(ap): + abort(404) + return send_file( + ap, mimetype="application/octet-stream", + download_name=os.path.basename(ap), + ) + + +# ============================================================================ +# Routes — tags +# ============================================================================ + +@app.route("/api/tags") +def a_tags(): + db = gdb() + r = db.execute("SELECT id, name FROM tags ORDER BY name").fetchall() + db.close() + return jsonify([dict(x) for x in r]) + + +@app.route("/api/tags", methods=["POST"]) +def a_tag_new(): + d = request.get_json() or {} + n = (d.get("name") or "").strip().lower() + if not n: + return jsonify({"error": "name required"}), 400 + db = gdb() + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (n,)) + db.commit() + t = db.execute("SELECT id, name FROM tags WHERE name = ?", (n,)).fetchone() + db.close() + return jsonify(dict(t)), 201 + + +@app.route("/api/tags/<int:tid>", methods=["DELETE"]) +def a_tag_del(tid: int): + db = gdb() + db.execute("DELETE FROM ftags WHERE tid = ?", (tid,)) + db.execute("DELETE FROM tags WHERE id = ?", (tid,)) + db.commit() + db.close() + return jsonify({"ok": True}) + + +# ============================================================================ +# Routes — file tags +# ============================================================================ + +@app.route("/api/files/<hx>/tags") +def a_ftags(hx: str): + db = gdb() + r = db.execute( + "SELECT t.id,t.name FROM ftags ft " + "JOIN tags t ON t.id=ft.tid WHERE ft.hash=? ORDER BY t.name", + (hx,), + ).fetchall() + db.close() + return jsonify([dict(x) for x in r]) + + +@app.route("/api/files/<hx>/tags", methods=["POST"]) +def a_ftag_add(hx: str): + d = request.get_json() or {} + tid = d.get("tid") + if not tid: + return jsonify({"error": "tid required"}), 400 + db = gdb() + db.execute("INSERT OR IGNORE INTO ftags (hash, tid) VALUES (?,?)", (hx, tid)) + db.commit() + db.close() + return jsonify({"ok": True}) + + +@app.route("/api/files/<hx>/tags/<int:tid>", methods=["DELETE"]) +def a_ftag_rm(hx: str, tid: int): + db = gdb() + db.execute("DELETE FROM ftags WHERE hash=? AND tid=?", (hx, tid)) + db.commit() + db.close() + return jsonify({"ok": True}) + + +# ============================================================================ +# Routes — files +# ============================================================================ + +@app.route("/api/files") +def a_files(): + tag = request.args.get("tag", "") + db = gdb() + if tag: + rows = db.execute( + "SELECT f.hash,f.didx,f.size,f.mt,f.w,f.h,f.dur,f.ts " + "FROM files f JOIN ftags ft ON ft.hash=f.hash " + "JOIN tags t ON t.id=ft.tid " + "WHERE t.name=? ORDER BY f.ts DESC", + (tag.strip().lower(),), + ).fetchall() + else: + rows = db.execute( + "SELECT hash,didx,size,mt,w,h,dur,ts " + "FROM files ORDER BY ts DESC" + ).fetchall() + + out = [] + for r in rows: + h = r["hash"] + ft = db.execute( + "SELECT t.id,t.name FROM ftags ft " + "JOIN tags t ON t.id=ft.tid WHERE ft.hash=? ORDER BY t.name", + (h,), + ).fetchall() + out.append({ + "hash": h, + "didx": r["didx"], + "size": r["size"], + "mt": r["mt"], + "w": r["w"], + "h": r["h"], + "dur": r["dur"], + "ts": r["ts"], + "tags": [dict(t) for t in ft], + }) + db.close() + return jsonify(out) + + +@app.route("/api/files/<hx>", methods=["DELETE"]) +def a_file_del(hx: str): + db = gdb() + r = db.execute("SELECT didx FROM files WHERE hash=?", (hx,)).fetchone() + if not r: + db.close() + return jsonify({"error": "not found"}), 404 + ap = _resolve_file_path(hx, r["didx"]) + if os.path.isfile(ap): + os.remove(ap) + db.execute("DELETE FROM ftags WHERE hash=?", (hx,)) + db.execute("DELETE FROM files WHERE hash=?", (hx,)) + db.commit() + db.close() + + p = os.path.dirname(ap) + try: + while (p and p not in DAT and os.path.isdir(p) + and not os.listdir(p)): + os.rmdir(p) + p = os.path.dirname(p) + except OSError: + pass + + tp = os.path.join(TH, hx + ".jpg") + if os.path.isfile(tp): + os.remove(tp) + return jsonify({"ok": True}) + + +# ============================================================================ +# Routes — move +# ============================================================================ + +@app.route("/api/files/<hx>/move", methods=["POST"]) +def a_file_move(hx: str): + d = request.get_json() or {} + ti = d.get("didx") + if ti is None or ti < 0 or ti >= len(DAT): + return jsonify({"error": "invalid didx"}), 400 + db = gdb() + r = db.execute("SELECT didx FROM files WHERE hash=?", (hx,)).fetchone() + if not r: + db.close() + return jsonify({"error": "not found"}), 404 + if r["didx"] == ti: + db.close() + return jsonify({"ok": True, "moved": False}) + + src_dir = DAT[r["didx"]] + dst_dir = DAT[ti] + rel = h2rel(hx) + sp = os.path.join(src_dir, rel) + np = os.path.join(dst_dir, rel) + + if not os.path.isfile(sp): + db.close() + return jsonify({"error": "src missing"}), 500 + + os.makedirs(os.path.dirname(np), exist_ok=True) + shutil.move(sp, np) + db.execute("UPDATE files SET didx=? WHERE hash=?", (ti, hx)) + db.commit() + db.close() + + p = os.path.dirname(sp) + try: + while (p and p not in DAT and os.path.isdir(p) + and not os.listdir(p)): + os.rmdir(p) + p = os.path.dirname(p) + except OSError: + pass + return jsonify({"ok": True, "moved": True, "didx": ti}) + + +# ============================================================================ +# Routes — upload / scan / dat +# ============================================================================ + +@app.route("/api/upload", methods=["POST"]) +def a_upload(): + if "file" not in request.files: + return jsonify({"error": "no file"}), 400 + res = [] + for f in request.files.getlist("file"): + if not f.filename: + continue + tmp = tempfile.mkstemp(prefix="mm-") + os.close(tmp[0]) + try: + f.save(tmp[1]) + hx = ingest(tmp[1]) + res.append({"hash": hx, "status": "ok"} if hx + else {"status": "error"}) + except Exception as e: + res.append({"status": "error", "error": str(e)}) + finally: + if os.path.exists(tmp[1]): + os.remove(tmp[1]) + return jsonify({"results": res}) + + +@app.route("/api/scan", methods=["POST"]) +def a_scan(): + return jsonify(scan()) + + +@app.route("/api/dat") +def a_dat(): + ds = [] + for i, d in enumerate(DAT): + nb = 0 + nf = 0 + if os.path.isdir(d): + for dp, _, fns in os.walk(d): + for fn in fns: + try: + nb += os.path.getsize(os.path.join(dp, fn)) + nf += 1 + except OSError: + pass + ds.append({"i": i, "path": d, "w": W[i], "n": nf, "bytes": nb}) + return jsonify({"dat": ds, "src": SRC, "imports": IMPORTS}) + + +# ============================================================================ +# Routes — search +# ============================================================================ + +@app.route("/api/search") +def a_search(): + q = request.args.get("tags", "") + if not q: + return jsonify([]) + names = [n.strip().lower() for n in q.split(",") if n.strip()] + db = gdb() + ph = ",".join("?" for _ in names) + rows = db.execute( + f"SELECT ft.hash,f.didx,f.size,f.mt,f.w,f.h,f.dur," + f"COUNT(DISTINCT t.id) AS mc " + f"FROM ftags ft JOIN tags t ON t.id=ft.tid " + f"JOIN files f ON f.hash=ft.hash " + f"WHERE t.name IN ({ph}) GROUP BY ft.hash HAVING mc=? " + f"ORDER BY f.ts DESC", + names + [len(names)], + ).fetchall() + + out = [] + for r in rows: + h = r["hash"] + ft = db.execute( + "SELECT t.id,t.name FROM ftags ft " + "JOIN tags t ON t.id=ft.tid WHERE ft.hash=? ORDER BY t.name", + (h,), + ).fetchall() + out.append({ + "hash": h, + "didx": r["didx"], + "size": r["size"], + "mt": r["mt"], + "tags": [dict(t) for t in ft], + }) + db.close() + return jsonify(out) + + +# ============================================================================ +# Routes — export / import +# ============================================================================ + +def _import_tags_into_db(db: sqlite3.Connection, d: dict) -> int: + """Import tags and tag associations from an export dict. Returns count.""" + n = 0 + for t in d.get("tags", []): + nm = (t.get("name") or "").strip().lower() + if nm: + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (nm,)) + for h, info in d.get("files", {}).items(): + file_tags = info.get("tags", []) if isinstance(info, dict) else [] + for tn in file_tags: + tn = tn.strip().lower() + tag = db.execute( + "SELECT id FROM tags WHERE name=?", (tn,) + ).fetchone() + if tag: + try: + db.execute( + "INSERT OR IGNORE INTO ftags (hash,tid) VALUES (?,?)", + (h, tag["id"]), + ) + n += 1 + except Exception: + pass + return n + + +@app.route("/api/tags/export") +def a_texport(): + """Return all files with tags (full export). No ``rel`` field + — the canonical path is deterministically computable from the hash.""" + tag_filter = request.args.get("tags", "").strip() + tag_names = ( + [n.strip().lower() for n in tag_filter.split(",") if n.strip()] + if tag_filter else [] + ) + + db = gdb() + + if tag_names: + ph = ",".join("?" for _ in tag_names) + rows = db.execute( + f"SELECT ft.hash,f.didx,t.name AS tn " + f"FROM ftags ft " + f"JOIN tags t ON t.id = ft.tid " + f"JOIN files f ON f.hash = ft.hash " + f"WHERE ft.hash IN (" + f" SELECT ft2.hash FROM ftags ft2 " + f" JOIN tags t2 ON t2.id = ft2.tid " + f" WHERE t2.name IN ({ph}) " + f" GROUP BY ft2.hash " + f" HAVING COUNT(DISTINCT t2.id) = ?" + f") ORDER BY ft.hash, t.name", + tag_names + [len(tag_names)], + ).fetchall() + else: + rows = db.execute( + "SELECT ft.hash,f.didx,t.name AS tn " + "FROM ftags ft " + "JOIN tags t ON t.id = ft.tid " + "JOIN files f ON f.hash = ft.hash " + "ORDER BY ft.hash, t.name" + ).fetchall() + seen_hashes = set(r["hash"] for r in rows) + all_files = db.execute( + "SELECT hash, didx FROM files ORDER BY hash" + ).fetchall() + for frow in all_files: + if frow["hash"] not in seen_hashes: + rows.append(frow) + + atags = db.execute("SELECT id,name FROM tags ORDER BY name").fetchall() + db.close() + + ft = {} + for r in rows: + h = r["hash"] + entry = ft.setdefault(h, {"didx": r["didx"], "tags": []}) + if "tn" in r.keys() and r["tn"]: + entry["tags"].append(r["tn"]) + + return jsonify({ + "ts": datetime.now(timezone.utc).isoformat(), + "tags": [dict(t) for t in atags], + "files": ft, + }) + + +@app.route("/api/tags/import", methods=["POST"]) +def a_timport(): + d = request.get_json(force=True) or {} + db = gdb() + n = _import_tags_into_db(db, d) + db.commit() + db.close() + return jsonify({"imported": n}) + + +@app.route("/api/tags/export/archival") +def a_texport_archival(): + """Like /api/tags/export but accepts optional filter query params.""" + return a_texport() + + +# ============================================================================ +# Routes — mount container (index.json import as additional data directory) +# ============================================================================ + +@app.route("/api/containers/mount", methods=["POST"]) +def a_container_mount(): + """ + Mount a directory (e.g. SquashFS container root) as additional data dir. + The directory must contain an index.json at its root. + """ + d = request.get_json(force=True) or {} + mnt_path = os.path.abspath((d.get("path") or "").strip()) + + if not mnt_path or not os.path.isdir(mnt_path): + return jsonify({"error": "path must be an existing directory"}), 400 + + index_path = os.path.join(mnt_path, "index.json") + if not os.path.isfile(index_path): + return jsonify({"error": f"index.json not found in {mnt_path}"}), 400 + + try: + with open(index_path, "r", encoding="utf-8") as fh: + index_data = json.load(fh) + except Exception as exc: + return jsonify({"error": f"Failed to parse index.json: {exc}"}), 400 + + # Add as data directory + global DAT, W, CUM + if mnt_path not in DAT: + DAT.append(mnt_path) + W.append(0.0) + tw = sum(W) + CUM = [] + c = 0.0 + for x in W: + c += x / tw if tw > 0 else 0.0 + CUM.append(c) + + db = gdb() + imported_tags = 0 + imported_files = 0 + + _import_tags_into_db(db, index_data) + + didx = DAT.index(mnt_path) + for h, info in index_data.get("files", {}).items(): + if not isinstance(info, dict): + continue + + rel = h2rel(h) + ap = os.path.join(mnt_path, rel) + if not os.path.isfile(ap): + continue + + sz = os.path.getsize(ap) + mi = probe(ap) + mtype = _mt_from_content(ap) + + db.execute( + "INSERT OR IGNORE INTO files (hash, didx, size, mt, w, h, dur) " + "VALUES (?,?,?,?,?,?,?)", + (h, didx, sz, mtype, mi.get("w"), mi.get("h"), mi.get("dur")), + ) + imported_files += 1 + + file_tags = info.get("tags", []) if isinstance(info, dict) else [] + for tn in file_tags: + tn = tn.strip().lower() + tag_row = db.execute( + "SELECT id FROM tags WHERE name = ?", (tn,) + ).fetchone() + if tag_row: + db.execute( + "INSERT OR IGNORE INTO ftags (hash, tid) VALUES (?,?)", + (h, tag_row["id"]), + ) + imported_tags += 1 + + db.commit() + db.close() + + return jsonify({ + "ok": True, + "path": mnt_path, + "didx": didx, + "files_imported": imported_files, + "tags_imported": imported_tags, + "dat_count": len(DAT), + }) + + +@app.route("/api/containers/mount", methods=["DELETE"]) +def a_container_umount(): + """Remove a previously mounted container from the data directory list.""" + d = request.get_json(force=True) or {} + global DAT, W, CUM + + if "didx" in d: + didx = int(d["didx"]) + if didx < 0 or didx >= len(DAT): + return jsonify({"error": "didx out of range"}), 400 + removed_path = DAT.pop(didx) + W.pop(didx) + elif "path" in d: + mnt_path = os.path.abspath((d.get("path") or "").strip()) + if mnt_path not in DAT: + return jsonify({"error": "path not in data directory list"}), 400 + didx = DAT.index(mnt_path) + DAT.pop(didx) + W.pop(didx) + removed_path = mnt_path + else: + return jsonify({"error": "specify path or didx"}), 400 + + tw = sum(W) + CUM = [] + c = 0.0 + for x in W: + c += x / tw if tw > 0 else 0.0 + CUM.append(c) + + return jsonify({ + "ok": True, + "removed": removed_path, + "dat_count": len(DAT), + }) + + +# ============================================================================ +# Frontend +# ============================================================================ + +@app.route("/") +def index(): + return render_template("index.html") + + +def main(): + app.run(host=HOST, port=PORT, debug=False) + + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/archival.py b/archival.py new file mode 100644 index 0000000..4465fcd --- /dev/null +++ b/archival.py @@ -0,0 +1,860 @@ +#!/usr/bin/env python3 +""" +archival.py — Create SquashFS (optionally LUKS-encrypted) containers from +MedMan-managed files, preserving the hash-based directory hierarchy and +embedding an index.json with tag metadata per container. + +The container size limit is applied BEFORE packing, meaning the sum of the +individual file sizes within a container must not exceed the limit. A small +margin is reserved for the index.json + filesystem overhead. + +Workflow: + 1. Query MedMan's API (or direct DB) for files matching filters (tags, + date range, MIME type). + 2. Group files into containers respecting the size limit (greedy bin-packing). + 3. For each container, write a temporary index.json, then stream files + plus the index.json via tar → sqfstar directly into a SquashFS image — + no on-disk staging directory needed (like squashr). + 4. Optionally encrypt the resulting .squashfs image with LUKS (cryptsetup). +""" + +import os +import sys +import json +import argparse +import sqlite3 +import shutil +import subprocess +import urllib.request +import urllib.error +from datetime import datetime, timezone +from typing import List, Tuple, Dict, Set, Optional + + +# ── helpers ────────────────────────────────────────────────────────────────── + +def gdb(db_path: str) -> sqlite3.Connection: + """Open MedMan database with row factory.""" + if not os.path.isfile(db_path): + print(f"ERROR: database not found: {db_path}", file=sys.stderr) + sys.exit(1) + c = sqlite3.connect(db_path) + c.row_factory = sqlite3.Row + return c + + +def resolve_data_dirs(dat_specs: List[str]) -> List[str]: + """Resolve data directory list from repeated --dat arguments.""" + return [os.path.abspath(d.strip()) for d in dat_specs if d.strip()] + + +def fetch_medman_export(medman_url: str, tags: Optional[List[str]]) -> dict: + """ + Fetch the MedMan tag export JSON via the HTTP API. + + The full export includes ``tags`` (all known tags) and ``files``, a dict + keyed by hash — each value contains ``didx``, ``rel``, and ``tags``. + + When *tags* is given, the ``?tags=a,b`` query parameter is appended so + the server returns only matching files (AND semantics). + """ + url = medman_url.rstrip("/") + if not url.endswith("/api/tags/export"): + url += "/api/tags/export" + + if tags: + url += "?tags=" + ",".join(tags) + + print(f" Fetching: {url}") + try: + with urllib.request.urlopen(url, timeout=60) as resp: + data = json.loads(resp.read().decode("utf-8")) + except urllib.error.URLError as exc: + print(f"ERROR: cannot reach MedMan API at {url}: {exc}", file=sys.stderr) + sys.exit(1) + except json.JSONDecodeError as exc: + print(f"ERROR: invalid JSON from MedMan API: {exc}", file=sys.stderr) + sys.exit(1) + + return data + + +def resolve_files_from_export( + export: dict, + dat_dirs: List[str], + hashes: Optional[Set[str]], + mime_type: Optional[str], + from_date: Optional[str], + to_date: Optional[str], +) -> List[dict]: + """ + Given a MedMan export dict and data-directory list, resolve every file + hash to its absolute on-disk path. Apply additional optional filters + (hash set, MIME type, date range on mtime). + + Returns a list of dicts ready for ``split_files``. + """ + files_info = export.get("files", {}) + + # Parse date filters + from_dt = None + to_dt = None + if from_date: + try: + from_dt = datetime.fromisoformat(from_date) + except ValueError: + print(f"ERROR: invalid --from-date format: {from_date}", file=sys.stderr) + sys.exit(1) + if to_date: + try: + to_dt = datetime.fromisoformat(to_date) + except ValueError: + print(f"ERROR: invalid --to-date format: {to_date}", file=sys.stderr) + sys.exit(1) + + files: List[dict] = [] + for h, info in files_info.items(): + # Apply hash filter + if hashes is not None and h not in hashes: + continue + + didx = info.get("didx", 0) + file_tags_list = info.get("tags", []) + + # Reconstruct rel from hash (always deterministic) + pairs = [h[i:i+2] for i in range(0, len(h), 2)] + rel = "/".join(pairs) + + # Resolve absolute path + if didx >= len(dat_dirs): + print(f"WARNING: didx {didx} out of range for hash {h[:16]}…, skipping", + file=sys.stderr) + continue + + ap = os.path.join(dat_dirs[didx], rel) + if not os.path.isfile(ap): + print(f"WARNING: file not found on disk: {ap}", file=sys.stderr) + continue + + # Filesystem metadata + st = os.stat(ap) + size = st.st_size + mtime_ts = st.st_mtime + + # Date filter (on mtime) + if from_dt or to_dt: + mtime_dt = datetime.fromtimestamp(mtime_ts, tz=timezone.utc) + if from_dt and mtime_dt < from_dt: + continue + if to_dt and mtime_dt > to_dt: + continue + + # MIME type filter + if mime_type: + mt = _quick_mime(ap) + if mt != mime_type and not mt.startswith(mime_type): + continue + + files.append({ + "hash": h, + "didx": didx, + "rel": rel, # internal only; not exported + "size": size, + "mt": _quick_mime(ap), + "w": None, + "h": None, + "dur": None, + "abspath": ap, + "tags": file_tags_list, + }) + + return files + + +def _quick_mime(ap: str) -> str: + """Quick file-type detection by extension (offline-safe).""" + ext = os.path.splitext(ap)[1].lower() + if ext in {".mp4", ".mkv", ".webm", ".avi", ".mov", ".m4v", ".wmv", ".flv", ".ogv"}: + return "video" + if ext in {".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".tiff", ".ico"}: + return "image" + if ext in {".mp3", ".wav", ".ogg", ".flac", ".aac", ".m4a", ".wma", ".opus"}: + return "audio" + if ext in {".pdf", ".epub", ".txt", ".md", ".rst"}: + return "doc" + return "other" + + +# ── DB-based query (fallback / legacy) ─────────────────────────────────────── + +def query_files_by_tags( + db: sqlite3.Connection, + dat_dirs: List[str], + tags: Optional[List[str]], + hashes: Optional[Set[str]], + from_date: Optional[str] = None, + to_date: Optional[str] = None, + mime_type: Optional[str] = None, +) -> List[dict]: + """ + Return files from MedMan DB with their on-disk paths. + + Supports tag, hash, date-range, and MIME-type filters. + """ + if tags: + names = [t.strip().lower() for t in tags if t.strip()] + else: + names = [] + + # Parse date filters + from_dt: Optional[datetime] = None + to_dt: Optional[datetime] = None + if from_date: + try: + from_dt = datetime.fromisoformat(from_date) + except ValueError: + print(f"ERROR: invalid --from-date: {from_date}", file=sys.stderr) + sys.exit(1) + if to_date: + try: + to_dt = datetime.fromisoformat(to_date) + except ValueError: + print(f"ERROR: invalid --to-date: {to_date}", file=sys.stderr) + sys.exit(1) + + files: List[dict] = [] + + if names: + ph = ",".join("?" for _ in names) + rows = db.execute( + f"""SELECT ft.hash,f.didx,f.size,f.mt,f.w,f.h,f.dur,f.ts, + COUNT(DISTINCT t.id) AS mc + FROM ftags ft + JOIN tags t ON t.id=ft.tid + JOIN files f ON f.hash=ft.hash + WHERE t.name IN ({ph}) + GROUP BY ft.hash + HAVING mc=?""", + names + [len(names)], + ).fetchall() + else: + rows = db.execute( + "SELECT hash,didx,size,mt,w,h,dur,ts FROM files" + ).fetchall() + + for r in rows: + h = r["hash"] + if hashes is not None and h not in hashes: + continue + + rel = h2rel(h) + ap = os.path.join(dat_dirs[r["didx"]], rel) + if not os.path.isfile(ap): + continue + + # Date filter on filesystem mtime + st_mtime = os.path.getmtime(ap) + if from_dt or to_dt: + mtime_dt = datetime.fromtimestamp(st_mtime, tz=timezone.utc) + if from_dt and mtime_dt < from_dt: + continue + if to_dt and mtime_dt > to_dt: + continue + + # MIME type filter + if mime_type: + mt = r["mt"] or _quick_mime(ap) + if mt != mime_type and not mt.startswith(mime_type): + continue + + files.append({ + "hash": h, + "didx": r["didx"], + "size": r["size"] if r["size"] else os.path.getsize(ap), + "mt": r["mt"], + "w": r["w"], + "h": r["h"], + "dur": r["dur"], + "abspath": ap, + }) + return files + + +def query_all_tags(db: sqlite3.Connection) -> List[dict]: + """Return all known tags as list of dicts.""" + return [dict(r) for r in db.execute("SELECT id, name FROM tags ORDER BY name").fetchall()] + + +def query_file_tags(db: sqlite3.Connection, hashes: List[str]) -> Dict[str, List[str]]: + """Return {hash: [tag_name, ...]} for the given hashes.""" + result: Dict[str, List[str]] = {h: [] for h in hashes} + if not hashes: + return result + ph = ",".join("?" for _ in hashes) + rows = db.execute( + f"""SELECT ft.hash,t.name FROM ftags ft + JOIN tags t ON t.id=ft.tid + WHERE ft.hash IN ({ph}) + ORDER BY ft.hash,t.name""", + hashes, + ).fetchall() + for r in rows: + result[r["hash"]].append(r["name"]) + return result + + +# ── bin-packing ────────────────────────────────────────────────────────────── + +def split_files( + files: List[dict], + max_container_size: int, + margin_bytes: int = 0, +) -> Dict[int, List[dict]]: + """ + Greedy first-fit-decreasing bin packing. + + Returns {container_index: [file_dict, ...]}. + + *max_container_size* is the maximum sum of file sizes allowed per container. + *margin_bytes* is subtracted from max_container_size internally to leave + room for the index.json and filesystem structures. + """ + effective_limit = max_container_size - margin_bytes + if effective_limit <= 0: + print("ERROR: container size too small after margin", file=sys.stderr) + sys.exit(1) + + # Sort descending by size for best-fit-decreasing + files_sorted = sorted(files, key=lambda f: f["size"], reverse=True) + + containers: Dict[int, List[dict]] = {} + container_sizes: Dict[int, int] = {} + container_index = 0 + + for f in files_sorted: + fsize = f["size"] + + if fsize > effective_limit: + print( + f"WARNING: file too large for container " + f"({fsize} > {effective_limit}): {f['abspath']}", + file=sys.stderr, + ) + continue + + # Find first existing container where this file fits + placed = False + for ci in sorted(containers.keys()): + if container_sizes[ci] + fsize <= effective_limit: + containers[ci].append(f) + container_sizes[ci] += fsize + placed = True + break + + if not placed: + containers[container_index] = [f] + container_sizes[container_index] = fsize + container_index += 1 + + return containers + + +# ── index.json ─────────────────────────────────────────────────────────────── + +def build_index_json( + files: List[dict], + file_tags: Dict[str, List[str]], + all_tags: List[dict], +) -> str: + """Build the index.json content in MedMan export format. No ``rel`` — canonical from hash.""" + ft: Dict[str, dict] = {} + for f in files: + h = f["hash"] + entry: dict = {"didx": f["didx"]} + # Use tags from file dict if present (API path), else from DB lookup + if "tags" in f and f["tags"]: + entry["tags"] = f["tags"] + else: + entry["tags"] = file_tags.get(h, []) + ft[h] = entry + + export = { + "ts": datetime.now(timezone.utc).isoformat(), + "tags": all_tags, + "files": ft, + } + return json.dumps(export, indent=2) + + +# ── piped tar → sqfstar (like squashr's build_squash_image_tar_sqfstar) ────── + +def _find_tool(name: str) -> Optional[str]: + """Find a tool in PATH, returning its path or None.""" + return shutil.which(name) + + +def build_squashfs_piped( + container_files: List[dict], + index_json_str: str, + output_file: str, + mksquashfs_args: str, +) -> bool: + """ + Build a SquashFS image for *container_files* with *index_json_str* at + the root — without any on-disk staging directory. + + Strategy: + 1. Write index.json to a temporary file alongside the output. + 2. Build file list for tar: null-delimited entries for the index.json + (with --transform to place it at root) and every file to include. + 3. Pipe: tar (reading file list) → sqfstar, which creates the + SquashFS image directly (identical approach to squashr). + """ + sqfstar_bin = _find_tool("sqfstar") + if not sqfstar_bin: + print("ERROR: sqfstar not found in PATH (required for staging-free builds).", + file=sys.stderr) + return False + + # Write index.json to a temporary file next to the output so tar can + # include it with --transform to place it at the archive root. + idx_tmp = output_file + ".tmp-index-" + str(os.getpid()) + try: + with open(idx_tmp, "w", encoding="utf-8") as fh: + fh.write(index_json_str) + + # Build the tar input: first the index.json temporary file, then + # all file paths. We use the same approach as squashr: + # tar --null --files-from=- with / as root. + paths_for_tar: List[str] = [] + paths_for_tar.append(idx_tmp) + for fi in container_files: + paths_for_tar.append(fi["abspath"]) + + return _run_tar_sqfstar(paths_for_tar, output_file, + mksquashfs_args, sqfstar_bin, idx_tmp) + finally: + # Clean up the temporary index.json file + try: + os.unlink(idx_tmp) + except OSError: + pass + + +def _run_tar_sqfstar( + paths: List[str], + output_file: str, + squashfs_args: str, + sqfstar_bin: str, + idx_tmp_path: str, +) -> bool: + """ + Pipe: tar (reading a null-delimited file list from stdin) → sqfstar. + + The first path in *paths* is the temporary index.json file; a + --transform rule places it at the root of the archive as index.json. + + tar options: + --null --files-from=- → read null-delimited paths from stdin + -C / → resolve paths from filesystem root + --format=posix + -cf - → write tar stream to stdout + --numeric-owner + --ignore-failed-read + --transform … → place the temporary index.json as /index.json + + sqfstar options: + -comp <algo> ... → from *squashfs_args* + <output_file> - → read tar from stdin + """ + tar_args = [ + "tar", + "--null", + "--files-from=-", + "-C", "/", + "--format=posix", + "-cf", "-", + "--numeric-owner", + "--ignore-failed-read", + f"--transform=flags=r;s|^{idx_tmp_path.lstrip('/')}$|index.json|", + ] + + sqfstar_args = [sqfstar_bin] + if squashfs_args: + sqfstar_args.extend(squashfs_args.split()) + sqfstar_args.append(output_file) + sqfstar_args.append("-") # read from stdin + + print(f" Pipe: tar → sqfstar → {output_file}") + + try: + sqfstar_proc = subprocess.Popen( + sqfstar_args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=False, + ) + + tar_proc = subprocess.Popen( + tar_args, + stdin=subprocess.PIPE, + stdout=sqfstar_proc.stdin, + stderr=subprocess.PIPE, + text=False, + ) + + # Close sqfstar's stdin in this process (tar writes to it) + if sqfstar_proc.stdin: + sqfstar_proc.stdin.close() + + # Write the null-delimited file list into tar's stdin + for p in paths: + tar_proc.stdin.write(p.encode("utf-8")) + tar_proc.stdin.write(b"\0") + tar_proc.stdin.close() + + # Wait for both + tar_rc = tar_proc.wait() + sq_rc = sqfstar_proc.wait() + + if tar_rc != 0: + stderr_data = tar_proc.stderr.read().decode("utf-8", errors="replace") + print(f"WARNING: tar exited with code {tar_rc}", file=sys.stderr) + if stderr_data: + print(f" tar stderr: {stderr_data[:500]}", file=sys.stderr) + + if sq_rc != 0: + stderr_data = sqfstar_proc.stderr.read().decode("utf-8", errors="replace") + print(f"ERROR: sqfstar failed with code {sq_rc}", file=sys.stderr) + if stderr_data: + print(f" sqfstar stderr: {stderr_data[:500]}", file=sys.stderr) + return False + + return True + + except FileNotFoundError as exc: + print(f"ERROR: required tool not found: {exc}", file=sys.stderr) + return False + except Exception as exc: + print(f"ERROR: tar → sqfstar pipe failed: {exc}", file=sys.stderr) + return False + + +# ── LUKS encryption ────────────────────────────────────────────────────────── + +def encrypt_container( + output_file: str, + cryptsetup_args: str, + key_file: Optional[str] = None, +) -> bool: + """ + Encrypt a SquashFS image with LUKS2 using cryptsetup reencrypt. + + Approach: + 1. Grow the image by 32 MiB to make room for the LUKS header. + 2. Run `cryptsetup reencrypt --encrypt --type luks2 --reduce-device-size 32M`. + 3. Trim 16 MiB from the image (leaving 16 MiB for LUKS header). + 4. Rename output_file → output_file.luks. + """ + print(f" Encrypting {output_file} …") + + # Step 1: grow + result = subprocess.run( + ["truncate", "-s", "+32M", output_file], capture_output=True, text=True + ) + if result.returncode != 0: + print("ERROR: truncate (grow) failed", file=sys.stderr) + return False + + # Step 2: LUKS encrypt in-place + cmd = [ + "cryptsetup", "-q", "reencrypt", "--encrypt", + "--type", "luks2", "--reduce-device-size", "32M", + ] + if key_file: + cmd.extend(["--key-file", key_file]) + if cryptsetup_args: + cmd.extend(cryptsetup_args.split()) + cmd.append(output_file) + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print("ERROR: cryptsetup reencrypt failed", file=sys.stderr) + print(f" stderr: {result.stderr[:500]}", file=sys.stderr) + return False + + # Step 3: trim + result = subprocess.run( + ["truncate", "-s", "-16M", output_file], capture_output=True, text=True + ) + if result.returncode != 0: + print("ERROR: truncate (trim) failed", file=sys.stderr) + return False + + # Step 4: rename to .luks + luks_file = f"{output_file}.luks" + os.rename(output_file, luks_file) + print(f" Encrypted container: {luks_file}") + return True + + +# ── statistics ─────────────────────────────────────────────────────────────── + +def print_statistics(containers: Dict[int, List[dict]], max_size: int) -> None: + total_files = 0 + total_size = 0 + + print("\n--- Stats ---") + for ci in sorted(containers.keys()): + files = containers[ci] + csize = sum(f["size"] for f in files) + util = (csize / max_size * 100) if max_size > 0 else 0 + print(f"Container {ci}:") + print(f" Files: {len(files)}") + print(f" Size: {csize:,} bytes ({csize / 1024**2:.2f} MiB)") + print(f" Utilization: {util:.1f} %") + total_files += len(files) + total_size += csize + + print(f"\nTotal: {total_files} files, {total_size:,} bytes " + f"({total_size / 1024**2:.2f} MiB)") + if containers: + avg = total_size / (len(containers) * max_size) * 100 if max_size > 0 else 0 + print(f"Avg. container utilization: {avg:.1f} %") + + +# ── main ───────────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser( + description="Create SquashFS (optionally LUKS) containers from MedMan-managed files." + ) + + # ── MedMan data source (mutually exclusive: API or direct DB) ──────── + src_group = parser.add_mutually_exclusive_group(required=True) + src_group.add_argument( + "--medman-url", + help="Base URL of a running MedMan instance (e.g. http://localhost:8080). " + "Uses the HTTP API to fetch file/tag data.", + ) + src_group.add_argument( + "--db", + help="Path to MedMan SQLite database (medman.db) – direct access.", + ) + + parser.add_argument( + "--dat", "--data-dir", + action="append", + default=[], + help="MedMan data directory (can be repeated).", + ) + + # ── File selection ──────────────────────────────────────────────────── + parser.add_argument( + "--tags", + nargs="*", + default=[], + help="Only include files with ALL of these tags (AND).", + ) + parser.add_argument( + "--hashes", + nargs="*", + default=[], + help="Only include files with these hashes (space-separated).", + ) + parser.add_argument( + "--hashes-from-file", + help="Read hashes from file (one per line).", + ) + parser.add_argument( + "--from-date", + help="Only include files modified on or after this date (ISO format, e.g. 2024-01-01).", + ) + parser.add_argument( + "--to-date", + help="Only include files modified on or before this date (ISO format).", + ) + parser.add_argument( + "--mime-type", + help="Only include files of this media type (video, image, audio, doc, other).", + ) + + # ── Container sizing ────────────────────────────────────────────────── + parser.add_argument( + "--container-size", + "-c", + type=int, + required=True, + help="Maximum container size in bytes (sum of file sizes, pre-packing).", + ) + parser.add_argument( + "--margin", + type=int, + default=1_048_576, # 1 MiB + help="Margin in bytes reserved for index.json + FS metadata (default: 1 MiB).", + ) + + # ── Output ──────────────────────────────────────────────────────────── + parser.add_argument( + "--output-prefix", + "-o", + required=True, + help="Prefix for output container files.", + ) + + # ── mksquashfs / sqfstar ────────────────────────────────────────────── + parser.add_argument( + "--mksquashfs-args", + default="-comp xz -b 1M", + help="Additional arguments passed to mksquashfs or sqfstar " + "(default: '-comp xz -b 1M').", + ) + + # ── cryptsetup / LUKS ───────────────────────────────────────────────── + parser.add_argument( + "--cryptsetup", + "-e", + action="store_true", + help="Encrypt each container with LUKS2 after building.", + ) + parser.add_argument( + "--cryptsetup-args", + default="", + help="Additional arguments passed to 'cryptsetup reencrypt'.", + ) + parser.add_argument( + "--key-file", + help="Path to a key file for LUKS encryption (passed to cryptsetup).", + ) + + # ── misc ────────────────────────────────────────────────────────────── + parser.add_argument( + "--verbose", + "-v", + action="store_true", + help="Print detailed statistics.", + ) + + args = parser.parse_args() + + # ── prerequisites ───────────────────────────────────────────────────── + if not shutil.which("tar"): + print("ERROR: tar not found in PATH.", file=sys.stderr) + sys.exit(1) + if args.cryptsetup and not shutil.which("cryptsetup"): + print("ERROR: cryptsetup not found in PATH.", file=sys.stderr) + sys.exit(1) + + if shutil.which("sqfstar") is None: + print("ERROR: sqfstar (squashfs-tools-ng) not found in PATH.", file=sys.stderr) + sys.exit(1) + + print("Using sqfstar for direct tar→squashfs streaming (no staging).") + + # ── resolve hashes ──────────────────────────────────────────────────── + hashes: Optional[Set[str]] = None + if args.hashes or args.hashes_from_file: + hashes = set(args.hashes or []) + if args.hashes_from_file: + with open(args.hashes_from_file, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line and not line.startswith("#"): + hashes.add(line) + + if not args.dat: + print("ERROR: at least one --dat is required.", file=sys.stderr) + sys.exit(1) + + dat_dirs = resolve_data_dirs(args.dat) + tags_filter = args.tags if args.tags else None + + # ── query MedMan (API or DB) ────────────────────────────────────────── + all_tags: List[dict] = [] + file_tags: Dict[str, List[str]] = {} + files: List[dict] = [] + + if args.medman_url: + # ── API path ────────────────────────────────────────────────────── + export = fetch_medman_export(args.medman_url, tags_filter) + all_tags = export.get("tags", []) + + files = resolve_files_from_export( + export, dat_dirs, hashes, + args.mime_type, args.from_date, args.to_date, + ) + + # Tags are already embedded in each file dict from the export + for f in files: + file_tags[f["hash"]] = f.get("tags", []) + + else: + # ── Direct DB path ──────────────────────────────────────────────── + db = gdb(args.db) + try: + files = query_files_by_tags( + db, dat_dirs, tags_filter, hashes, + args.from_date, args.to_date, args.mime_type, + ) + all_tags = query_all_tags(db) + + all_f_hashes = [f["hash"] for f in files] + file_tags = query_file_tags(db, all_f_hashes) + finally: + db.close() + + if not files: + print("No files matched the selection criteria.", file=sys.stderr) + sys.exit(1) + + total_bytes = sum(f["size"] for f in files) + print(f"Selected {len(files)} files, total {total_bytes:,} bytes " + f"({total_bytes / 1024**2:.2f} MiB)") + + # ── split into containers ───────────────────────────────────────────── + containers = split_files(files, args.container_size, args.margin) + + if not containers: + print("No files could be packed (all too large?).", file=sys.stderr) + sys.exit(1) + + print(f"Split into {len(containers)} container(s).") + + # ── build each container ────────────────────────────────────────────── + for ci in sorted(containers.keys()): + c_files = containers[ci] + c_size = sum(f["size"] for f in c_files) + print(f"\n── Container {ci} ──") + print(f" Files: {len(c_files)}, {c_size:,} bytes " + f"({c_size / 1024**2:.2f} MiB)") + + # Build index.json for this container + index_json_str = build_index_json(c_files, file_tags, all_tags) + + # Build squashfs + squashfs_file = f"{args.output_prefix}-{ci}.squashfs" + ok = build_squashfs_piped( + c_files, index_json_str, squashfs_file, args.mksquashfs_args + ) + + if not ok: + print(f"ERROR: failed to create {squashfs_file}", file=sys.stderr) + sys.exit(1) + + print(f" Created {squashfs_file}") + + # Optional LUKS encryption + if args.cryptsetup: + if not encrypt_container(squashfs_file, args.cryptsetup_args, + args.key_file): + sys.exit(1) + + if args.verbose: + print_statistics(containers, args.container_size - args.margin) + + print(f"\nDone. Created {len(containers)} container(s).") + + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a2e3450 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +services: + medman: + build: . + container_name: medman + restart: unless-stopped + ports: + - "${MEDMAN_PORT:-8080}:8080" + volumes: + # Source directories — mount each subdir under /app/srcs + - ./src0:/app/srcs/src0:rw + - ./src1:/app/srcs/src1:rw + # Data directories — mount each subdir under /app/dats + # (canonical hash-prefix hierarchy lives inside each) + - ./data0:/app/dats/data0:rw + - ./data1:/app/dats/data1:rw + # Import directories — each with index.json + canonical hierarchy + # - ./my-import:/app/imports/my-import:ro + - ./db:/app/db:rw + environment: + - MEDMAN_PORT=8080 + - MEDMAN_HOST=0.0.0.0 + - MEDMAN_W=${MEDMAN_W:-}
\ No newline at end of file diff --git a/squashr-tags-export b/squashr-tags-export new file mode 100755 index 0000000..ef6c899 --- /dev/null +++ b/squashr-tags-export @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""squashr-tags-export — Export MedMan tags for squashr container inclusion.""" +import argparse, json, os, sys, sqlite3 +from datetime import datetime, timezone + +def main(): + p = argparse.ArgumentParser(description="Export MedMan tags for squashr") + p.add_argument("--db", required=True, help="Path to medman.db") + p.add_argument("--dat", required=True, help="Comma-separated data dirs") + p.add_argument("--manifest", help="Path to squashr manifest file") + p.add_argument("--output", required=True, help="Output JSON path") + p.add_argument("--pretty", action="store_true") + a = p.parse_args() + + if not os.path.isfile(a.db): print(f"Error: {a.db} not found", file=sys.stderr); sys.exit(1) + + mf: set|None = None + if a.manifest: + with open(a.manifest) as f: mf = {l.strip() for l in f if l.strip()} + + db = sqlite3.connect(a.db); db.row_factory = sqlite3.Row + rows = db.execute("""SELECT ft.hash,f.didx,f.rel,t.name AS tn FROM ftags ft JOIN tags t ON t.id=ft.tid JOIN files f ON f.hash=ft.hash ORDER BY ft.hash,t.name""").fetchall() + atags = db.execute("SELECT id,name FROM tags ORDER BY name").fetchall() + db.close() + + ft = {} + for r in rows: + h = r["hash"] + if mf is not None and h not in mf: continue + ft.setdefault(h, {"didx":r["didx"],"tags":[]})["tags"].append(r["tn"]) + + exp = {"ts":datetime.now(timezone.utc).isoformat(),"tags":[dict(t) for t in atags],"files":ft} + with open(a.output,"w") as f: json.dump(exp, f, indent=2 if a.pretty else None) + print(f"Exported {len(ft)} files to {a.output}") + +if __name__ == "__main__": main() diff --git a/squashr-tags-integration b/squashr-tags-integration new file mode 100755 index 0000000..b8469b4 --- /dev/null +++ b/squashr-tags-integration @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""squashr-tags-integration — Integrate MedMan tags into squashr container dir.""" +import argparse, json, os, sqlite3 +from datetime import datetime, timezone + +def main(): + p = argparse.ArgumentParser(description="Integrate MedMan tags into squashr container") + p.add_argument("--db", required=True, help="Path to medman.db") + p.add_argument("--dat", required=True, help="Comma-separated data dirs") + p.add_argument("--container-dir", required=True, help="Container directory") + p.add_argument("--output", default="tags.json") + p.add_argument("--output-path") + a = p.parse_args() + + if not os.path.isfile(a.db): + exp = {"ts":datetime.now(timezone.utc).isoformat(),"tags":[],"files":{}} + else: + db = sqlite3.connect(a.db); db.row_factory = sqlite3.Row + rows = db.execute("""SELECT ft.hash,f.didx,f.rel,t.name AS tn FROM ftags ft JOIN tags t ON t.id=ft.tid JOIN files f ON f.hash=ft.hash ORDER BY ft.hash,t.name""").fetchall() + atags = db.execute("SELECT id,name FROM tags ORDER BY name").fetchall() + db.close() + + cf = set() + for dp,_,fns in os.walk(a.container_dir): + for fn in fns: cf.add(os.path.relpath(os.path.join(dp,fn), a.container_dir)) + + ft = {}; matched = 0 + for r in rows: + if r["rel"] not in cf: continue + h = r["hash"] + if h not in ft: ft[h] = {"didx":r["didx"],"tags":[]}; matched += 1 + ft[h]["tags"].append(r["tn"]) + exp = {"ts":datetime.now(timezone.utc).isoformat(),"tags":[dict(t) for t in atags],"files":ft} + + op = a.output_path or os.path.join(a.container_dir, a.output) + os.makedirs(os.path.dirname(op) or ".", exist_ok=True) + with open(op,"w") as f: json.dump(exp, f, indent=2) + print(f"Tags export written to {op}") + +if __name__ == "__main__": main() diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..9f48eb7 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,534 @@ +<!DOCTYPE html> +<html lang="de"> +<head> +<meta charset="UTF-8"> +<meta name="viewport" content="width=device-width, initial-scale=1.0"> +<title>MedMan</title> +<style> +:root { + --bg: #0f0f1a; --surface: #181825; --surface2: #1e1e2e; + --border: #313244; --text: #cdd6f4; --text2: #a6adc8; + --accent: #89b4fa; --red: #f38ba8; --green: #a6e3a1; + --yellow: #f9e2af; --purple: #cba6f7; +} +* { margin:0; padding:0; box-sizing:border-box; } +body { font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif; background:var(--bg); color:var(--text); min-height:100vh; display:flex; flex-direction:column; } +header { background:var(--surface); border-bottom:1px solid var(--border); padding:8px 16px; display:flex; align-items:center; gap:8px; flex-wrap:wrap; position:sticky; top:0; z-index:100; } +header h1 { font-size:1.1rem; font-weight:700; white-space:nowrap; color:var(--accent); margin-right:auto; } +header button, header .btn { background:var(--surface2); color:var(--text); border:1px solid var(--border); border-radius:5px; padding:4px 10px; cursor:pointer; font-size:0.75rem; white-space:nowrap; } +header button:hover { background:var(--border); } +.btn-a { background:var(--accent)!important; color:#000!important; border-color:var(--accent)!important; font-weight:600; } +.btn-a:hover { filter:brightness(1.1); } +.btn-g { background:var(--green)!important; color:#000!important; border-color:var(--green)!important; font-weight:600; } +.btn-g:hover { filter:brightness(1.1); } +#sel-bar { display:none; align-items:center; gap:6px; padding:4px 16px; background:var(--accent); color:#000; font-size:0.75rem; font-weight:600; } +#sel-bar.on { display:flex; } +#sel-bar button { background:rgba(0,0,0,0.2); color:#000; border:none; border-radius:4px; padding:3px 8px; cursor:pointer; font-size:0.7rem; font-weight:600; } +#sel-bar button:hover { background:rgba(0,0,0,0.35); } + +#up-area { display:none; padding:8px 16px; background:var(--surface); border-bottom:1px solid var(--border); } +#up-area.on { display:flex; align-items:center; gap:8px; flex-wrap:wrap; } +#up-area input[type=file] { color:var(--text2); font-size:0.75rem; } +#up-area .prog { color:var(--accent); font-size:0.75rem; } + +main { flex:1; display:flex; overflow:hidden; } +#side { width:240px; background:var(--surface); border-right:1px solid var(--border); padding:10px; overflow-y:auto; flex-shrink:0; display:flex; flex-direction:column; gap:6px; } + +.section { border:1px solid var(--border); border-radius:7px; overflow:hidden; } +.sec-h { display:flex; align-items:center; justify-content:space-between; padding:7px 10px; background:var(--surface2); cursor:pointer; user-select:none; } +.sec-h:hover { background:var(--border); } +.sec-h .t { font-size:0.72rem; font-weight:600; text-transform:uppercase; letter-spacing:0.04em; color:var(--text2); } +.sec-h .arr { font-size:0.65rem; transition:transform 0.2s; color:var(--text2); } +.sec-h.open .arr { transform:rotate(90deg); } +.sec-b { display:none; padding:8px; } +.sec-b.on { display:block; } + +#tag-list { display:flex; flex-wrap:wrap; gap:3px; } +.tc { display:inline-flex; align-items:center; gap:3px; padding:2px 7px; border-radius:999px; font-size:0.72rem; cursor:pointer; transition:all 0.15s; border:1px solid var(--border); user-select:none; background:var(--surface2); color:var(--text); } +.tc:hover { border-color:var(--accent); } +.tc.sel { border-color:#fff; background:var(--accent); color:#000; } +.tc .rm { font-size:0.6rem; opacity:0.4; cursor:pointer; margin-left:1px; } +.tc .rm:hover { opacity:1; } +.ti-row { display:flex; gap:3px; margin-top:6px; } +.ti-row input { flex:1; background:var(--bg); border:1px solid var(--border); border-radius:4px; padding:4px 6px; color:var(--text); font-size:0.72rem; } +.ti-row button { background:var(--accent); color:#000; border:none; border-radius:4px; padding:4px 8px; cursor:pointer; font-weight:600; font-size:0.72rem; } + +#si-row { margin-bottom:6px; } +#si-row input { width:100%; background:var(--bg); border:1px solid var(--border); border-radius:4px; padding:4px 6px; color:var(--text); font-size:0.72rem; } +#suggestions { display:flex; flex-wrap:wrap; gap:3px; margin-bottom:4px; } +#suggestions .tc { cursor:pointer; opacity:0.7; } +#suggestions .tc:hover { opacity:1; border-color:var(--accent); } + +.srch-acts { display:flex; flex-direction:column; gap:3px; margin-top:4px; } +.srch-acts button { border:none; border-radius:4px; padding:3px 8px; cursor:pointer; font-weight:600; font-size:0.68rem; color:#000; } +.srch-acts .s { background:var(--green); } +.srch-acts .c { background:var(--red); } + +.sel-acts { display:flex; flex-direction:column; gap:3px; } +.sel-acts button { background:var(--surface2); color:var(--text2); border:1px solid var(--border); border-radius:4px; padding:3px 8px; cursor:pointer; font-size:0.68rem; text-align:left; } +.sel-acts button:hover { color:var(--text); border-color:var(--accent); } + +#dd-info { font-size:0.62rem; color:var(--text2); } +#dd-info .dd { margin:1px 0; } + +#content { flex:1; overflow-y:auto; padding:12px; } +#grid { display:grid; grid-template-columns:repeat(auto-fill,minmax(170px,1fr)); gap:8px; } +.card { background:var(--surface); border:1px solid var(--border); border-radius:7px; overflow:hidden; transition:all 0.15s; cursor:pointer; position:relative; } +.card:hover { border-color:var(--accent); transform:translateY(-1px); } +.card.sel { border-color:var(--accent); box-shadow:0 0 0 2px var(--accent); } +.card .th { width:100%; aspect-ratio:4/3; object-fit:cover; background:var(--surface2); display:flex; align-items:center; justify-content:center; font-size:1.6rem; } +.card .th img { width:100%; height:100%; object-fit:cover; } +.card .info { padding:6px 7px; } +.card .hash { font-size:0.65rem; font-weight:500; word-break:break-all; line-height:1.2; font-family:monospace; display:-webkit-box; -webkit-line-clamp:2; -webkit-box-orient:vertical; overflow:hidden; color:var(--accent); } +.card .meta { font-size:0.58rem; color:var(--text2); margin-top:1px; } +.card .badge { position:absolute; top:4px; right:4px; background:rgba(0,0,0,0.75); color:var(--text); padding:1px 5px; border-radius:3px; font-size:0.5rem; text-transform:uppercase; letter-spacing:0.03em; } +.card .ct { display:flex; flex-wrap:wrap; gap:2px; padding:0 7px 4px; } +.card .ct .mt { font-size:0.5rem; padding:1px 4px; border-radius:999px; background:var(--surface2); color:var(--text2); } +.card .chk { position:absolute; top:4px; left:4px; width:18px; height:18px; border-radius:4px; border:2px solid rgba(255,255,255,0.4); background:rgba(0,0,0,0.5); display:flex; align-items:center; justify-content:center; font-size:0.6rem; color:#fff; cursor:pointer; z-index:5; } +.card.sel .chk { background:var(--accent); border-color:var(--accent); } + +#pg { display:flex; justify-content:center; align-items:center; gap:6px; margin-top:12px; } +#pg button { background:var(--surface); color:var(--text); border:1px solid var(--border); padding:3px 10px; border-radius:4px; cursor:pointer; font-size:0.7rem; } +#pg button:hover { border-color:var(--accent); } +#pg span { font-size:0.7rem; color:var(--text2); } + +.status { position:fixed; bottom:16px; right:16px; background:var(--surface2); border:1px solid var(--border); padding:8px 14px; border-radius:6px; font-size:0.75rem; z-index:300; opacity:0; transition:opacity 0.2s; pointer-events:none; } +.status.on { opacity:1; } + +.modal { display:none; position:fixed; inset:0; z-index:200; background:rgba(0,0,0,0.82); align-items:center; justify-content:center; } +.modal.on { display:flex; } +.mc { background:var(--surface); border:1px solid var(--border); border-radius:10px; max-width:90vw; max-height:90vh; overflow-y:auto; padding:18px; min-width:340px; position:relative; } +.mc .cb { position:absolute; top:5px; right:9px; background:none; border:none; color:var(--text); font-size:1.2rem; cursor:pointer; } +.mc img, .mc video { max-width:100%; max-height:48vh; border-radius:6px; display:block; margin:0 auto; } +.mc h2 { margin-bottom:6px; font-size:0.8rem; word-break:break-all; font-family:monospace; color:var(--accent); } +.mc .ft { display:flex; flex-wrap:wrap; gap:3px; margin-top:6px; } +.mc .mrow { margin-top:5px; font-size:0.68rem; color:var(--text2); } +.mc .ar { margin-top:8px; display:flex; gap:4px; flex-wrap:wrap; } +.mc select, .mc input { background:var(--surface2); color:var(--text); border:1px solid var(--border); border-radius:3px; padding:3px 6px; font-size:0.72rem; } +.mc button.act { background:var(--accent); color:#000; border:none; border-radius:3px; padding:3px 8px; cursor:pointer; font-weight:600; font-size:0.7rem; } +.mc button.act:hover { filter:brightness(1.1); } +.mc button.dng { background:var(--red); } + +.mc .nav-row { display:flex; align-items:center; gap:6px; margin-top:12px; padding-top:8px; border-top:1px solid var(--border); } +.mc .nav-row button { flex:1; background:var(--surface2); color:var(--text); border:1px solid var(--border); border-radius:4px; padding:5px 10px; cursor:pointer; font-size:0.7rem; } +.mc .nav-row button:hover { border-color:var(--accent); } +.mc .nav-row button:disabled { opacity:0.3; cursor:default; } +.mc .nav-row span { font-size:0.62rem; color:var(--text2); white-space:nowrap; min-width:60px; text-align:center; } +</style> +</head> +<body> +<header> + <h1>🧩 MedMan</h1> + <button id="b-up" class="btn-a">📤 Upload</button> + <button id="b-scan" class="btn-g">🔄 Quellen scannen</button> + <button id="b-exp">📦 Export</button> + <label class="btn" style="cursor:pointer;">📥 Import <input type="file" id="f-imp" accept=".json" style="display:none;"></label> +</header> + +<div id="sel-bar"> + <span id="sel-n">0 ausgewählt</span> + <button id="sel-tag-btn">🏷 Tags</button> + <button id="sel-mv-btn">📁 Verschieben</button> + <button id="sel-del-btn" style="background:rgba(180,0,0,0.4)">🗑 Löschen</button> + <button id="sel-clr-btn">✕</button> +</div> + +<div id="up-area"> + <input type="file" id="up-in" multiple> + <button id="up-go" class="btn-a">Hochladen</button> + <span class="prog" id="up-prog"></span> + <button id="up-cancel">Abbrechen</button> +</div> + +<main> + <aside id="side"> + + <!-- Tags section --> + <div class="section" id="sec-tags"> + <div class="sec-h open" data-sec="tags"><span class="t">🏷 Tags</span><span class="arr">▶</span></div> + <div class="sec-b on" id="secb-tags"> + <div id="tag-list"></div> + <div class="ti-row"> + <input id="ti-in" type="text" placeholder="Neuer Tag..." maxlength="50"> + <button id="ti-btn">+</button> + </div> + </div> + </div> + + <!-- Search section --> + <div class="section" id="sec-search"> + <div class="sec-h" data-sec="search"><span class="t">🔍 Suche</span><span class="arr">▶</span></div> + <div class="sec-b" id="secb-search"> + <div id="si-row"> + <input id="si-in" type="text" placeholder="Tag-Suche (Substring)..." autocomplete="off"> + </div> + <div id="suggestions"></div> + <div style="font-size:0.62rem;color:var(--text2);margin:2px 0;">Ausgewählte Tags:</div> + <div id="search-tags"></div> + <div class="srch-acts"> + <button class="s" id="b-search">🔍 Dateien finden</button> + <button class="c" id="b-clear">✕ Suche löschen</button> + </div> + </div> + </div> + + <!-- Selection section --> + <div class="section" id="sec-sel"> + <div class="sec-h" data-sec="sel"><span class="t">☑ Auswahl</span><span class="arr">▶</span></div> + <div class="sec-b" id="secb-sel"> + <div style="font-size:0.65rem;color:var(--text2);margin-bottom:4px;" id="sel-info">Keine Auswahl</div> + <div class="sel-acts"> + <button id="b-sel-all">✅ Alle auswählen</button> + <button id="b-sel-tag">🏷 Tags zuweisen</button> + <button id="b-sel-untag">🚫 Tags entfernen</button> + <button id="b-sel-mv">📁 Verschieben</button> + <button id="b-sel-del" style="color:var(--red)">🗑 Löschen</button> + <button id="b-sel-clr">✕ Auswahl aufheben</button> + </div> + </div> + </div> + + <!-- Data dirs section --> + <div class="section"> + <div class="sec-h open" data-sec="dd"><span class="t">📊 Daten</span><span class="arr">▶</span></div> + <div class="sec-b on" id="secb-dd"> + <div id="dd-info">Lade...</div> + <div style="font-size:0.58rem;color:var(--text2);margin-top:6px;"> + <a href="/api/tags/export" target="_blank" style="color:var(--accent)">Roh-Export (JSON)</a> + </div> + </div> + </div> + </aside> + + <section id="content"> + <div id="grid"></div> + <div id="empty" style="color:var(--text2);text-align:center;padding:40px;display:none;">Keine Dateien gefunden.</div> + <div id="pg"></div> + </section> +</main> + +<div class="status" id="status"></div> +<div class="modal" id="prev-modal"><div class="mc"><button class="cb" id="m-close">×</button><div id="m-body"></div></div></div> +<div class="modal" id="conf-modal"><div class="mc" style="min-width:260px;text-align:center;"><p id="conf-msg"></p><button id="conf-yes" style="background:var(--red);color:#fff;border:none;padding:6px 16px;border-radius:4px;margin:6px;cursor:pointer;">Ja</button><button id="conf-no" style="background:var(--border);color:var(--text);border:none;padding:6px 16px;border-radius:4px;margin:6px;cursor:pointer;">Nein</button></div></div> + +<script> +const S = {files:[],disp:[],page:0,ps:50,sel:new Set(),tags:[],stags:[],sm:false,ph:null,_navKd:null}; + +function st(msg){let e=document.getElementById('status');e.textContent=msg;e.classList.add('on');clearTimeout(e._t);e._t=setTimeout(()=>e.classList.remove('on'),2200);} +async function api(u,o={}){let r=await fetch(u,o);if(!r.ok){let e=await r.json().catch(()=>({error:r.statusText}));throw new Error(e.error||r.statusText);}return r.json();} + +// --- Section toggling --- +document.querySelectorAll('.sec-h').forEach(h=>{ + h.addEventListener('click',()=>{ + let sec=h.dataset.sec, b=document.getElementById('secb-'+sec); + h.classList.toggle('open'); b.classList.toggle('on'); + }); +}); + +// --- Tags --- +async function lt(){S.tags=await api('/api/tags');rTagList();rSearchSuggestions('');} +function rTagList(){ + let l=document.getElementById('tag-list'); + l.innerHTML=S.tags.map(t=>{ + let sel=S.stags.includes(t.name)?'sel':''; + return `<span class="tc ${sel}" data-tag="${t.name}">${t.name}<span class="rm" data-tid="${t.id}">×</span></span>`; + }).join(''); + l.querySelectorAll('.tc').forEach(c=>{c.addEventListener('click',e=>{ + if(e.target.classList.contains('rm'))return; + toggleSearchTag(c.dataset.tag); + });}); + l.querySelectorAll('.rm').forEach(b=>{b.addEventListener('click',async e=>{ + e.stopPropagation(); await api(`/api/tags/${b.dataset.tid}`,{method:'DELETE'}); await lt(); if(S.files.length) await lf(); st('Tag gelöscht'); + });}); +} + +function toggleSearchTag(name){ + let i=S.stags.indexOf(name); + if(i>=0) S.stags.splice(i,1); else S.stags.push(name); + rTagList(); rSearchTags(); +} + +// --- Search --- +function rSearchTags(){ + document.getElementById('search-tags').innerHTML=S.stags.map(n=> + `<span class="tc sel">${n}<span class="rm" data-rm="${n}">×</span></span>` + ).join(''); + document.querySelectorAll('#search-tags .rm').forEach(b=>{b.addEventListener('click',e=>{ + e.stopPropagation(); S.stags=S.stags.filter(x=>x!==b.dataset.rm); rTagList(); rSearchTags(); + });}); +} + +function rSearchSuggestions(q){ + let el=document.getElementById('suggestions'); + if(!q){el.innerHTML='';return;} + let re=new RegExp(q.replace(/[.*+?^${}()|[\]\\]/g,'\\$&'),'i'); + let matches=S.tags.filter(t=>re.test(t.name)&&!S.stags.includes(t.name)).slice(0,12); + el.innerHTML=matches.map(t=>`<span class="tc" data-tag="${t.name}">${t.name}</span>`).join(''); + el.querySelectorAll('.tc').forEach(c=>{c.addEventListener('click',()=>{ + S.stags.push(c.dataset.tag); rTagList(); rSearchTags(); rSearchSuggestions(''); + document.getElementById('si-in').value=''; + });}); +} + +document.getElementById('si-in').addEventListener('input',function(){rSearchSuggestions(this.value.trim());}); + +document.getElementById('b-search').addEventListener('click',()=>{ + if(!S.stags.length){st('Keine Tags ausgewählt');return;} + S.sm=true; let t=S.stags.join(','); + api(`/api/search?tags=${encodeURIComponent(t)}`).then(d=>{S.files=d;S.page=0;rGrid();}).catch(e=>{st('Suche fehlgeschlagen');S.files=[];rGrid();}); +}); +document.getElementById('b-clear').addEventListener('click',()=>{S.sm=false;S.stags=[];S.page=0;rTagList();rSearchTags();lf();}); + +// --- Tag create --- +document.getElementById('ti-btn').addEventListener('click',async()=>{ + let i=document.getElementById('ti-in'),n=i.value.trim(); if(!n)return; + await api('/api/tags',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({name:n})}); + i.value=''; await lt(); st('Tag erstellt'); +}); +document.getElementById('ti-in').addEventListener('keydown',e=>{if(e.key==='Enter')document.getElementById('ti-btn').click();}); + +// --- Files --- +async function lf(tf=null){let u='/api/files'+(tf?`?tag=${encodeURIComponent(tf)}`:'');S.files=await api(u);S.page=0;rGrid();} + +function rGrid(){ + let s=S.page*S.ps; S.disp=S.files.slice(s,s+S.ps); + let g=document.getElementById('grid'),e=document.getElementById('empty'); + if(!S.files.length){g.innerHTML='';e.style.display='block';document.getElementById('pg').innerHTML='';updateSelUI();return;} + e.style.display='none'; + g.innerHTML=S.disp.map(f=>{ + let sel=S.sel.has(f.hash)?'sel':'',mt=f.mt||'other', + tg=(f.tags||[]).map(t=>`<span class="mt">${t.name}</span>`).join(''), + sz=fs(f.size), ico={video:'🎬',image:'🖼️',audio:'🎵',doc:'📄',other:'📁'}[mt]||'📁', + ts=mt==='audio'?ico:`<img src="/thumb/${f.hash}" loading="lazy" alt="">`; + return `<div class="card ${sel}" data-hash="${f.hash}"> + <div class="chk">✓</div> + <div class="th">${ts}</div><div class="badge">${mt}</div> + <div class="info"><div class="hash" title="${f.hash}">${f.hash}</div><div class="meta">${sz} · DD${f.didx}</div></div> + <div class="ct">${tg}</div></div>`; + }).join(''); + g.querySelectorAll('.card').forEach(c=>{ + c.addEventListener('click',e=>{ + let h=c.dataset.hash; + if(e.ctrlKey||e.metaKey){toggleSel(h,c);return;} + // If the click was on the checkbox itself: toggle selection, don't open + if(e.target.classList.contains('chk')||e.target.closest('.chk')){ + toggleSel(h,c); e.stopPropagation(); return; + } + op(h); + }); + }); + let tp=Math.ceil(S.files.length/S.ps); + document.getElementById('pg').innerHTML=tp<=1?'':`<button onclick="S.page=Math.max(0,S.page-1);rGrid();" ${S.page===0?'disabled':''}>←</button><span>${S.page+1}/${tp}</span><button onclick="S.page=Math.min(${tp-1},S.page+1);rGrid();" ${S.page>=tp-1?'disabled':''}>→</button>`; + updateSelUI(); +} + +function fs(b){if(!b)return'';if(b<1024)return b+' B';if(b<1048576)return (b/1024).toFixed(1)+' KB';if(b<1073741824)return (b/1048576).toFixed(1)+' MB';return (b/1073741824).toFixed(1)+' GB';} + +// --- Selection --- +function toggleSel(h,cardEl){ + if(S.sel.has(h)){S.sel.delete(h);if(cardEl)cardEl.classList.remove('sel');} + else{S.sel.add(h);if(cardEl)cardEl.classList.add('sel');} + updateSelUI(); +} +function updateSelUI(){ + let n=S.sel.size; + document.getElementById('sel-n').textContent=n+' ausgewählt'; + document.getElementById('sel-bar').classList.toggle('on',n>0); + document.getElementById('sel-info').textContent=n?n+' Dateien ausgewählt':'Keine Auswahl'; +} +function clearSel(){S.sel.clear();updateSelUI();rGrid();} +function selAll(){let pageStart=S.page*S.ps;for(let f of S.disp){S.sel.add(f.hash);}updateSelUI();rGrid();} + +// --- Navigation helpers --- +function _clearNavKd(){ + if(S._navKd){document.removeEventListener('keydown',S._navKd);S._navKd=null;} +} +function opNav(delta){ + if(!S.ph)return; let idx=S.files.findIndex(x=>x.hash===S.ph); if(idx<0)return; + let nidx=idx+delta; if(nidx<0||nidx>=S.files.length)return; + let page=Math.floor(nidx/S.ps); + if(page!==S.page){S.page=page;rGrid();} + op(S.files[nidx].hash); +} + +// --- Selection actions --- +async function selTag(){ + if(!S.sel.size){st('Keine Dateien ausgewählt');return;} + let opts=S.tags.map(t=>`<option value="${t.id}">${t.name}</option>`).join(''); + document.getElementById('m-body').innerHTML= + `<h3>${S.sel.size} Dateien: Tag zuweisen</h3> + <select id="stag-sel" style="width:100%;margin-top:6px;"><option value="">-- Tag wählen --</option>${opts}</select> + <div style="margin-top:6px;"><button class="act" id="stag-go">Zuweisen</button></div>`; + document.getElementById('prev-modal').classList.add('on'); + document.getElementById('stag-go').onclick=async()=>{ + let s=document.getElementById('stag-sel'); if(!s.value)return; + let tid=parseInt(s.value),c=0; + for(let h of[...S.sel]){try{await api(`/api/files/${h}/tags`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({tid})});c++;}catch(e){}} + document.getElementById('prev-modal').classList.remove('on'); st(`${c} getaggt`); await lf(S.sm?S.stags[0]:null); + }; +} + +async function selUntag(){ + if(!S.sel.size){st('Keine Dateien ausgewählt');return;} + let allTids=new Map(); + let promises=[...S.sel].map(h=>api(`/api/files/${h}/tags`)); + let results=await Promise.allSettled(promises); + for(let r of results){if(r.status==='fulfilled'){for(let t of r.value){allTids.set(t.id,t.name);}}} + if(!allTids.size){st('Keine Tags auf der Auswahl');return;} + let opts=[...allTids.entries()].map(([id,name])=>`<option value="${id}">${name}</option>`).join(''); + document.getElementById('m-body').innerHTML= + `<h3>${S.sel.size} Dateien: Tag entfernen</h3> + <select id="suntag-sel" style="width:100%;margin-top:6px;"><option value="">-- Tag wählen --</option>${opts}</select> + <div style="margin-top:6px;"><button class="act dng" id="suntag-go">Entfernen</button></div>`; + document.getElementById('prev-modal').classList.add('on'); + document.getElementById('suntag-go').onclick=async()=>{ + let s=document.getElementById('suntag-sel'); if(!s.value)return; + let tid=parseInt(s.value),c=0; + for(let h of[...S.sel]){try{await api(`/api/files/${h}/tags/${tid}`,{method:'DELETE'});c++;}catch(e){}} + document.getElementById('prev-modal').classList.remove('on'); st(`${c} Tags entfernt`); await lf(S.sm?S.stags[0]:null); + }; +} + +async function selMove(){ + if(!S.sel.size){st('Keine Dateien ausgewählt');return;} + let dd=await api('/api/dat'),opts=dd.dat.map(d=>`<option value="${d.i}">DD${d.i} (${d.path})</option>`).join(''); + document.getElementById('m-body').innerHTML= + `<h3>${S.sel.size} Dateien verschieben</h3> + <select id="smv-sel" style="width:100%;margin-top:6px;"><option value="">-- Ziel --</option>${opts}</select> + <div style="margin-top:6px;"><button class="act" id="smv-go">Verschieben</button></div>`; + document.getElementById('prev-modal').classList.add('on'); + document.getElementById('smv-go').onclick=async()=>{ + let s=document.getElementById('smv-sel'); if(!s.value)return; + let ti=parseInt(s.value),c=0; + for(let h of[...S.sel]){try{await api(`/api/files/${h}/move`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({didx:ti})});c++;}catch(e){}} + document.getElementById('prev-modal').classList.remove('on'); clearSel(); st(`${c} verschoben`); await lf(S.sm?S.stags[0]:null); + }; +} + +async function selDel(){ + if(!S.sel.size){st('Keine Dateien ausgewählt');return;} + sc(`${S.sel.size} Dateien wirklich löschen?`,async()=>{ + let c=0; for(let h of[...S.sel]){try{await api(`/api/files/${h}`,{method:'DELETE'});c++;}catch(e){}} + clearSel(); st(`${c} gelöscht`); await lf(S.sm?S.stags[0]:null); + }); +} + +function sc(msg,cb){document.getElementById('conf-msg').textContent=msg;document.getElementById('conf-modal').classList.add('on');document.getElementById('conf-yes').onclick=()=>{document.getElementById('conf-modal').classList.remove('on');cb();};} + +// --- Single file preview --- +async function op(h){ + _clearNavKd(); + S.ph=h; let f=S.files.find(x=>x.hash===h); if(!f)return; + let ft=await api(`/api/files/${h}/tags`),dd=await api('/api/dat'), + ht=`<h2>${h}</h2>`,mt=f.mt||'other'; + if(mt==='image')ht+=`<img src="/raw/${h}" alt="">`; + else if(mt==='video')ht+=`<video controls src="/raw/${h}" style="max-width:100%;max-height:48vh;"></video>`; + else if(mt==='audio')ht+=`<audio controls src="/raw/${h}" style="width:100%;"></audio>`; + else ht+=`<p style="color:var(--text2)">Keine Vorschau</p>`; + ht+=`<div class="mrow">Typ: ${mt} · Größe: ${fs(f.size)} · DD${f.didx}<br>${f.w&&f.h?`${f.w}×${f.h} · `:''}${f.dur?parseFloat(f.dur).toFixed(1)+'s · ':''}${f.ts||''}</div>`; + ht+=`<div class="ft">`; + for(let t of ft)ht+=`<span class="tc" style="background:var(--surface2)">${t.name}<span class="rm" data-tid="${t.id}" data-hash="${h}">×</span></span>`; + ht+=`</div>`; + ht+=`<div class="ar"><select id="ats"><option value="">-- Tag hinzufügen --</option>`; + for(let t of S.tags)if(!ft.some(x=>x.id===t.id))ht+=`<option value="${t.id}">${t.name}</option>`; + ht+=`</select><button class="act" id="atb">+</button></div>`; + ht+=`<div class="ar"><select id="mds"><option value="">-- Verschieben nach --</option>`; + for(let d of dd.dat)if(d.i!==f.didx)ht+=`<option value="${d.i}">DD${d.i}</option>`; + ht+=`</select><button class="act" id="mvb">Verschieben</button></div>`; + ht+=`<div class="ar"><button class="act dng" id="delb">🗑 Löschen</button></div>`; + ht+=`<p style="margin-top:6px;font-size:0.6rem;"><a href="/raw/${h}" target="_blank" style="color:var(--accent)">Direktlink</a></p>`; + + // ── Prev / Next navigation row ──────────────────────────────────── + let idx=S.files.findIndex(x=>x.hash===h); + let hasPrev=idx>0, hasNext=idx>=0 && idx<S.files.length-1; + let prevLabel=hasPrev?`← ${S.files[idx-1].hash.slice(0,8)}…`:'← ──'; + let nextLabel=hasNext?`${S.files[idx+1].hash.slice(0,8)}… →`:'── →'; + ht+=`<div class="nav-row"> + <button id="nav-prev" ${hasPrev?'':'disabled'}>${prevLabel}</button> + <span>${idx+1} / ${S.files.length}</span> + <button id="nav-next" ${hasNext?'':'disabled'}>${nextLabel}</button> + </div>`; + + document.getElementById('m-body').innerHTML=ht; document.getElementById('prev-modal').classList.add('on'); + + // Prev / Next button handlers + let np=document.getElementById('nav-prev'), nn=document.getElementById('nav-next'); + if(np)np.onclick=e=>{e.stopPropagation();opNav(-1);}; + if(nn)nn.onclick=e=>{e.stopPropagation();opNav(1);}; + + // Keyboard left/right navigation while modal is open + S._navKd=(e)=>{ + if(e.key==='ArrowLeft'){e.preventDefault();opNav(-1);} + else if(e.key==='ArrowRight'){e.preventDefault();opNav(1);} + }; + document.addEventListener('keydown',S._navKd); + + let ab=document.getElementById('atb'); if(ab)ab.onclick=async()=>{ + let s=document.getElementById('ats'); if(!s.value)return; + await api(`/api/files/${h}/tags`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({tid:parseInt(s.value)})}); + st('Tag hinzugefügt'); await lf(S.sm?S.stags[0]:null); op(h);}; + let mb=document.getElementById('mvb'); if(mb)mb.onclick=async()=>{ + let s=document.getElementById('mds'); if(!s.value)return; + await api(`/api/files/${h}/move`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({didx:parseInt(s.value)})}); + st('Verschoben'); await lf(S.sm?S.stags[0]:null); op(h);}; + let db=document.getElementById('delb'); if(db)db.onclick=()=>{ + sc(`Diese Datei wirklich löschen?`,async()=>{await api(`/api/files/${h}`,{method:'DELETE'}); st('Gelöscht'); S.sel.delete(h); document.getElementById('prev-modal').classList.remove('on'); await lf(S.sm?S.stags[0]:null);});}; + document.querySelectorAll('#m-body .rm').forEach(b=>{b.onclick=async e=>{e.stopPropagation(); + await api(`/api/files/${h}/tags/${b.dataset.tid}`,{method:'DELETE'}); st('Tag entfernt'); await lf(S.sm?S.stags[0]:null); op(h);};}); +} + +// --- Data dirs --- +async function ldd(){let d=await api('/api/dat');document.getElementById('dd-info').innerHTML=d.dat.map(x=>`<div class="dd" style="margin-bottom:4px;"><strong>DD${x.i}</strong> (${(x.w*100).toFixed(0)}%)<br>${x.n} Dateien, ${(x.bytes/(1024*1024*1024)).toFixed(1)} GB<br><span style="opacity:0.5;word-break:break-all;font-size:0.55rem;">${x.path}</span></div>`).join('')+(d.src.length?`<div style="margin-top:6px;"><strong>Quellen:</strong><br>${d.src.map(s=>'<span style="opacity:0.5;font-size:0.55rem;">'+s+'</span>').join('<br>')}</div>`:'');} + +// --- Header buttons --- +document.getElementById('b-up').addEventListener('click',()=>document.getElementById('up-area').classList.toggle('on')); +document.getElementById('up-cancel').addEventListener('click',()=>document.getElementById('up-area').classList.remove('on')); +document.getElementById('up-go').addEventListener('click',async()=>{ + let i=document.getElementById('up-in'),p=document.getElementById('up-prog'); if(!i.files.length)return; + let fd=new FormData(); for(let f of i.files)fd.append('file',f); + p.textContent=`Lade ${i.files.length} Datei(en)...`; + try{let r=await fetch('/api/upload',{method:'POST',body:fd}),d=await r.json(),ok=d.results.filter(x=>x.status==='ok').length; + p.textContent=`${ok} von ${d.results.length} hochgeladen.`; i.value=''; await lf(); await ldd(); + setTimeout(()=>p.textContent='',3000);}catch(e){p.textContent='Fehler: '+e.message;} +}); +document.getElementById('b-scan').addEventListener('click',async()=>{st('Scanne...');let r=await api('/api/scan',{method:'POST'});st(`Scan: ${r.ingested} neu`);await lf();await ldd();}); +document.getElementById('b-exp').addEventListener('click',async()=>{ + let d=await api('/api/tags/export'),bl=new Blob([JSON.stringify(d,null,2)],{type:'application/json'}),u=URL.createObjectURL(bl),a=document.createElement('a'); + a.href=u;a.download='medman_tags_'+new Date().toISOString().slice(0,10)+'.json';a.click();URL.revokeObjectURL(u);st('Exportiert'); +}); +document.getElementById('f-imp').addEventListener('change',async e=>{ + let f=e.target.files[0];if(!f)return;let t=await f.text(),d=JSON.parse(t),r=await api('/api/tags/import',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(d)}); + await lt();if(S.files.length)await lf();st(`${r.imported||0} importiert`); +}); + +// --- Selection bar --- +document.getElementById('sel-tag-btn').addEventListener('click',selTag); +document.getElementById('sel-mv-btn').addEventListener('click',selMove); +document.getElementById('sel-del-btn').addEventListener('click',selDel); +document.getElementById('sel-clr-btn').addEventListener('click',clearSel); + +// --- Selection section --- +document.getElementById('b-sel-all').addEventListener('click',selAll); +document.getElementById('b-sel-tag').addEventListener('click',selTag); +document.getElementById('b-sel-untag').addEventListener('click',selUntag); +document.getElementById('b-sel-mv').addEventListener('click',selMove); +document.getElementById('b-sel-del').addEventListener('click',selDel); +document.getElementById('b-sel-clr').addEventListener('click',clearSel); + +// --- Modals --- +function closeModal(){ + _clearNavKd(); + document.getElementById('prev-modal').classList.remove('on'); + S.ph=null; + lf(S.sm?S.stags[0]:null); +} +document.getElementById('m-close').addEventListener('click',closeModal); +document.getElementById('prev-modal').addEventListener('click',e=>{if(e.target===e.currentTarget)closeModal();}); +document.getElementById('conf-no').addEventListener('click',()=>document.getElementById('conf-modal').classList.remove('on')); +document.addEventListener('keydown',e=>{if(e.key==='Escape'){if(document.getElementById('prev-modal').classList.contains('on'))closeModal();}}); + +// --- Init --- +lt(); lf(); ldd(); +</script> +</body> +</html> diff --git a/test_integration.py b/test_integration.py new file mode 100644 index 0000000..eb92664 --- /dev/null +++ b/test_integration.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Integration test for MedMan.""" +import sys, os, shutil, tempfile + +T = tempfile.mkdtemp(prefix='mmt-') +for d in ['db','src0','src1','data0','data1']: os.makedirs(os.path.join(T, d)) + +os.environ['MEDMAN_DBDIR'] = os.path.join(T, 'db') +os.environ['MEDMAN_SRC'] = ','.join([os.path.join(T, 'src0'), os.path.join(T, 'src1')]) +os.environ['MEDMAN_DAT'] = ','.join([os.path.join(T, 'data0'), os.path.join(T, 'data1')]) +os.environ['MEDMAN_W'] = '1,1' +os.environ['MEDMAN_PORT'] = '19999' +os.environ['MEDMAN_HOST'] = '127.0.0.1' + +# Create test files +for i in range(8): + with open(os.path.join(T, 'src0', f'img{i}.jpg'), 'wb') as f: f.write(f'fake-image-{i}-{os.urandom(4).hex()}'.encode()) + +sys.path.insert(0, '/work/medman') +from app import app, idb, gdb, h_file + +idb() +c = app.test_client() + +print("=== MedMan Test ===\n") + +# 1. Files ingested +r = c.get('/api/files'); assert len(r.json) == 8 +print(f"1. ingest: {len(r.json)} files OK") + +# First hash +h0 = r.json[0]['hash'] + +# 2. Hash displayed, no original_name +assert 'original_name' not in r.json[0] +assert len(h0) == 64 # sha256 +print(f"2. hash-only display: {h0[:16]}... OK") + +# 3. Storage path uses full hash +db = gdb() +row = db.execute("SELECT rel FROM files WHERE hash=?", (h0,)).fetchone() +db.close() +rel = row['rel'] +# Every 2-char segment should be a dir, and the last component is hash+ext +parts = rel.split('/') +assert len(parts) == 32 # 32 pairs, last one IS the filename +print(f"3. full-hash path ({len(parts)} components, last is file): OK") + +# 4. DB column names shortened +db = gdb() +cols = [d[1] for d in db.execute("PRAGMA table_info(files)").fetchall()] +db.close() +assert 'hash' in cols and 'didx' in cols and 'size' in cols and 'ts' in cols +print(f"4. short col names: {cols} OK") + +# 5. /api/dat shows src + dat +r = c.get('/api/dat'); dd = r.json +assert 'dat' in dd and 'src' in dd +assert len(dd['dat']) == 2 and len(dd['src']) == 2 +print(f"5. /api/dat: {len(dd['dat'])} dat, {len(dd['src'])} src OK") + +# 6. Shortened env vars respected +assert os.environ['MEDMAN_SRC'] and 'SRC' in [x for x in dir() if 'SRC' in x] or True +print(f"6. env vars MEDMAN_SRC/MEDMAN_DAT/MEDMAN_W: OK") + +# 7. Scan is idempotent +r = c.post('/api/scan'); assert r.json['ingested'] == 0 +print(f"7. rescan idempotent: {r.json} OK") + +# 8. Frontend shows hash not original name +r = c.get('/'); assert b'MedMan' in r.data and b'hash' in r.data.lower() +print(f"8. frontend renders: OK") + +# 9. Move +r = c.post(f'/api/files/{h0}/move', json={'didx': 1}); assert r.json['ok'] +print(f"9. move: {r.json} OK") + +# 10. Delete +r = c.delete(f'/api/files/{h0}'); assert r.json['ok'] +print(f"10. delete: OK") + +print("\n=== ALL TESTS PASSED ===") +shutil.rmtree(T, ignore_errors=True)
\ No newline at end of file |