commit files

decorrupt_files.py

@@ -0,0 +1,112 @@
+
+
+import psycopg2
+import subprocess
+import os
+import shlex
+
+def get_conn():
+
+    conn = psycopg2.connect(
+        host="postgres",
+        database="mydatabase",
+        user="myuser",
+        password="mypassword",
+        port="5432"  # Default PostgreSQL port
+    )
+
+
+    return conn
+
+def process_file(file):
+
+    filename = file[0]
+    ref_x_status = file[1]
+    ref_12_status = file[2]
+    ref_16_status = file[3]
+    ## process ref_x
+    if ref_x_status < 2:
+        process_ref(filename, "ref_x.MP4")
+    ## process ref_12
+    if ref_12_status < 2:
+         process_ref(filename, "ref_12.MP4")
+    # ### process ref_16
+    # if ref_16_status < 2:
+    #     process_ref(filename, "ref_16.MP4")
+
+
+def update_table(filename, ref, status):
+    
+    conn = get_conn()
+
+    try:
+        cursor = conn.cursor()
+        cursor.execute(f"UPDATE public.decorrupt SET {ref} = {status} WHERE filename = '{filename}'")
+        conn.commit()
+    except psycopg2.Error as e:
+        print(f"Error connecting to or querying the database: {e}")
+    finally:
+        # 6. Close Cursor and Connection
+        if cursor:
+            cursor.close()
+        if conn:
+            conn.close()
+
+
+def process_ref(filename, ref):
+    print(f"processing file:{filename} ref:{ref}")
+    
+    #set to processing
+    update_table(filename, ref.replace(".MP4", ""), 1)
+    
+    # execute docker commnad
+    execute_docker(filename, ref)
+
+    # rename file
+    os.rename(f"/motherload/scan/{filename}_fixed-s1.MP4", f"/motherload/scan/{filename}_fixed-s1_{ref}")
+
+    #set to processed
+    update_table(filename, ref.replace(".MP4", ""), 2)
+
+
+def execute_docker(filename, ref):
+    motherload_path = "/home/geezo/01_projects/decorrupt/motherload"
+    command = ["docker", "run", "--rm", "-v", f"{motherload_path}:/motherload", "untrunc", "-s", f"/motherload/reference/{ref}", f"/motherload/scan/{filename}"
+
+    ]
+
+    try:
+        result = subprocess.run(command, 
+                                capture_output=True, text=True, check=True)
+        print(result.stdout)
+    except subprocess.CalledProcessError as e:
+        print(f"Error executing Docker command: {e}")
+        print(f"Stderr: {e.stderr}")
+
+    
+    print(f"executing docker... {command}")
+
+
+def get_files(conn):
+    try:
+        cursor = conn.cursor()
+        cursor.execute(f"SELECT * FROM public.decorrupt WHERE isresolved = 0 and isdeleted = 0 ORDER BY filename")
+        rows = cursor.fetchall()
+    except psycopg2.Error as e:
+        print(f"Error connecting to or querying the database: {e}")
+    finally:
+        # 6. Close Cursor and Connection
+        if cursor:
+            cursor.close()
+        if conn:
+            conn.close()
+    return rows
+
+
+def main():
+    files = get_files(get_conn())
+    for file in files:
+        process_file(file)
+
+if __name__ == "__main__":
+    main()

docker-compose.yaml

@@ -0,0 +1,83 @@
+
+services:
+  orchestrator:
+    build:
+      context: .
+      dockerfile: Dockerfile 
+    image: orchestrator:latest
+    container_name: decorrupt_orchestrator
+    restart: always
+    privileged: true
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: postgres://myuser:mypassword@postgres:5432/mydatabase
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./motherload:/motherload
+
+  postgres:
+    image: postgres:15
+    container_name: postgres_db
+    restart: always
+    environment:
+      POSTGRES_USER: myuser
+      POSTGRES_PASSWORD: mypassword
+      POSTGRES_DB: mydatabase
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U myuser -d mydatabase"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+
+  pgadmin:
+    image: dpage/pgadmin4:latest
+    container_name: pgadmin
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@example.com
+      PGADMIN_DEFAULT_PASSWORD: admin
+    ports:
+      - "8080:80"
+    depends_on:
+      - postgres
+    volumes:
+      - pgadmin_data:/var/lib/pgadmin
+
+  streamlit:
+    image: decorrupt_viz:latest   # <-- your built/pushed streamlit image
+    container_name: streamlit_app
+    restart: always
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      DB_HOST: postgres
+      DB_PORT: 5432
+      DB_NAME: mydatabase
+      DB_USER: myuser
+      DB_PASSWORD: mypassword
+      TABLE_NAME: "public.decorrupt"   # change to your actual table
+    ports:
+      - "8501:8501"
+  
+  filebrowser:
+    image: filebrowser/filebrowser:latest
+    container_name: filebrowser
+    restart: unless-stopped
+    ports:
+      - "8081:80"            # access at http://localhost:8081
+    volumes:
+      - ./motherload/scan:/srv          # folder you want to browse
+      - ./filebrowser.db:/database.db
+      - ./filebrowser.json:/config.json
+
+volumes:
+  postgres_data:
+  pgadmin_data:

docker_command.sh

@@ -0,0 +1,2 @@
+docker run --user 1000 -v ./motherload/playground:/vids -v ./motherload/reference:/refs untrunc -s /refs/ref_x.MP4 /vids/993065877504.mp4
+

dockerfile

@@ -0,0 +1,22 @@
+# Use an official lightweight Python image
+FROM python:3-alpine
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Copy requirements file into the container
+COPY requirements.txt .
+
+RUN apk add --no-cache \
+      postgresql-libs \
+      docker-cli \
+    && apk add --no-cache --virtual .build-deps \
+      gcc musl-dev postgresql-dev \
+    && python3 -m pip install --no-cache-dir -r requirements.txt \
+    && apk del .build-deps
+
+# Copy the rest of your project files into the container
+COPY decorrupt_files.py main.py
+
+# Run your Python file (replace main.py with your script)
+CMD ["python", "main.py"]

dockge/data/db-config.json

@@ -0,0 +1,3 @@
+{
+    "type": "sqlite"
+}

dockge/docker-compose.yml

@@ -0,0 +1,22 @@
+services:
+  dockge:
+    image: louislam/dockge:1
+    restart: unless-stopped
+    ports:
+      # Host Port : Container Port
+      - 5001:5001
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./data:/app/data
+        
+      # If you want to use private registries, you need to share the auth file with Dockge:
+      # - /root/.docker/:/root/.docker
+
+      # Stacks Directory
+      # ⚠️ READ IT CAREFULLY. If you did it wrong, your data could end up writing into a WRONG PATH.
+      # ⚠️ 1. FULL path only. No relative path (MUST)
+      # ⚠️ 2. Left Stacks Path === Right Stacks Path (MUST)
+      - /opt/stacks:/opt/stacks
+    environment:
+      # Tell Dockge where is your stacks directory
+      - DOCKGE_STACKS_DIR=/opt/stacks

move_files.py

@@ -0,0 +1,9 @@
+import os
+
+directory = "/home/geezo/01_projects/decorrupt/motherload/scan"
+contents = os.listdir(directory)
+
+for item in contents:
+    if "lost" in item:
+        filepath = f"{directory}/{item}"
+        os.rename(filepath, filepath.replace(" ", "_"))

prime_database.py

@@ -0,0 +1,33 @@
+import psycopg2
+import os
+
+conn = psycopg2.connect(
+    host="localhost",
+    database="mydatabase",
+    user="myuser",
+    password="mypassword",
+    port="5432"  # Default PostgreSQL port
+)
+
+
+directory_path = "/home/geezo/01_projects/decorrupt/motherload/scan"  # Replace with your directory
+all_entries = os.listdir(directory_path)
+
+files = [entry for entry in all_entries if os.path.isfile(os.path.join(directory_path, entry))]
+#print(files)
+
+
+try:
+    cursor = conn.cursor()
+    for file in files:
+        cursor.execute(f"INSERT INTO public.decorrupt (filename, ref_x, ref_12, ref_16) VALUES('{file}', 0, 0, 0)")
+        
+    conn.commit()
+except psycopg2.Error as e:
+    print(f"Error connecting to or querying the database: {e}")
+finally:
+    # 6. Close Cursor and Connection
+    if cursor:
+        cursor.close()
+    if conn:
+        conn.close()

requirements.txt

@@ -0,0 +1 @@
+psycopg2

streamlit/app.py

@@ -0,0 +1,202 @@
+# app.py
+# Minimal Streamlit status board using ONLY built-in components (no Altair).
+# - Connects directly to Postgres
+# - Orders rows: ORDER BY ref_x DESC, ref_12 DESC, ref_16 DESC
+# - Fixed column order: ref_x, ref_12, ref_16
+# - Each "dot" is a clickable link button to your File Browser URL
+# - Two generic action buttons per row (function placeholders provided)
+
+import os
+from urllib.parse import quote
+
+import pandas as pd
+import streamlit as st
+from sqlalchemy import create_engine, text
+
+# -----------------------------
+# Configuration
+# -----------------------------
+st.set_page_config(page_title="File Processing Status", layout="wide")
+
+# Hide chrome for a minimal look
+st.markdown(
+    """
+    <style>
+      header {visibility: hidden;}
+      #MainMenu {visibility: hidden;}
+      footer {visibility: hidden;}
+      .block-container {padding-top: 1rem; padding-bottom: 2rem;}
+      .grid-header {font-weight: 600; opacity: 0.8; padding: .25rem 0;}
+      .fname {white-space: nowrap; overflow: hidden; text-overflow: ellipsis;}
+    </style>
+    """,
+    unsafe_allow_html=True,
+)
+
+# DB config (fill via st.secrets or env)
+DB_HOST = "postgres"
+DB_PORT = "5432"
+DB_NAME = "mydatabase"
+DB_USER = "myuser"
+DB_PASSWORD = "mypassword"
+TABLE_NAME = "decorrupt"
+# Optional: restrict columns; leave empty to auto-detect
+REF_PREFIX = "ref_"
+
+# File Browser base (adjust to your host/IP)
+FILEBROWSER_BASE = "http://192.168.1.229:8081/files"
+
+# Expected ref columns (fixed order)
+REF_COLS = ["ref_x", "ref_12", "ref_16"]
+
+# Dot icons for statuses (built-in emoji; no custom CSS/JS)
+DOTS = {
+    -1: "🔴",  # unknown/error
+     0: "⚪",  # not processing (gray/white)
+     1: "🟠",  # processing (orange)
+     2: "🟢",  # processed (green)
+}
+
+LABELS = {0: "not processing", 1: "processing", 2: "processed", -1: "unknown"}
+
+
+def update_status(filename: str, column_name: str, new_value: int) -> bool:
+    """
+    Update a single column for a given filename in the specified table.
+
+    Args:
+        filename (str): The filename to update.
+        table_name (str): The table where the row exists.
+        column_name (str): The column to update (e.g., 'ref_x').
+        new_value (int): The new status value (e.g., 0, 1, 2).
+
+    Returns:
+        bool: True if update succeeded, False otherwise.
+    """
+    sql = text(f'UPDATE "decorrupt" SET "{column_name}" = :new_value WHERE filename = :filename')
+
+    try:
+        engine = get_engine()  # reuse your existing get_engine()
+        with engine.begin() as conn:  # auto-commits or rolls back
+            result = conn.execute(sql, {"new_value": new_value, "filename": filename})
+        return result.rowcount > 0
+    except Exception as e:
+        st.error(f"Failed to update {filename}.{column_name}: {e}")
+        return False
+# -----------------------------
+# DB helpers
+# -----------------------------
+def get_engine():
+    url = f"postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+    return create_engine(url, pool_pre_ping=True)
+
+
+def load_data(table_name: str) -> pd.DataFrame:
+    query = text(f'SELECT * FROM "{table_name}" WHERE isdeleted = 0 ORDER BY filename')
+    with get_engine().connect() as conn:
+        return pd.read_sql(query, conn)
+
+# -----------------------------
+# Actions (function placeholders)
+# -----------------------------
+def resolve_record(filename: str):
+    update_status(filename = filename, column_name = "isresolved", new_value = 1)
+    st.toast(f"{filename} resolved")
+
+
+def delete_record(filename: str):
+    update_status(filename = filename, column_name = "isdeleted", new_value = 1)
+    st.toast(f"{filename} deleted")
+
+
+# -----------------------------
+# Load & validate
+# -----------------------------
+st.title("File Processing Status")
+st.caption("0 = not processing · 1 = processing · 2 = processed")
+
+if "<YOUR_" in DB_NAME or "<YOUR_" in DB_USER or "<YOUR_" in DB_PASSWORD or "<YOUR_" in TABLE_NAME:
+    st.warning("Fill in DB credentials (DB_HOST/PORT/NAME/USER/PASSWORD) and TABLE_NAME via env or st.secrets, then rerun.")
+    st.stop()
+
+try:
+    df = load_data(TABLE_NAME)
+except Exception as e:
+    st.error(f"Could not load data: {e}")
+    st.stop()
+
+if "filename" not in df.columns:
+    st.error('Expected a "filename" column in the table.')
+    st.stop()
+
+present_refs = [c for c in REF_COLS if c in df.columns]
+if len(present_refs) != len(REF_COLS):
+    st.warning(f"Missing expected columns: {set(REF_COLS) - set(present_refs)}")
+
+# Ensure numeric statuses; map unknowns to -1
+for c in present_refs:
+    df[c] = pd.to_numeric(df[c], errors="coerce").fillna(-1).astype(int)
+
+# Preserve DB-defined ordering
+filenames_in_order = df["filename"].tolist()
+
+# -----------------------------
+# Legend (built-ins only)
+# -----------------------------
+with st.container():
+    l1, l2, l3, l4 = st.columns([1, 1, 1, 1])
+    l1.write(f'{DOTS[0]} 0 · {LABELS[0]}')
+    l2.write(f'{DOTS[1]} 1 · {LABELS[1]}')
+    l3.write(f'{DOTS[2]} 2 · {LABELS[2]}')
+    l4.write(f'{DOTS[-1]} -1 · {LABELS[-1]}')
+
+st.divider()
+
+# -----------------------------
+# Header row
+# -----------------------------
+with st.container():
+    c = st.columns([6, 2, 2, 2, 3])  # filename | ref_x | ref_12 | ref_16 | actions
+    c[0].markdown('<div class="grid-header">filename</div>', unsafe_allow_html=True)
+    c[1].markdown(f'<div class="grid-header">{REF_COLS[0]}</div>', unsafe_allow_html=True)
+    c[2].markdown(f'<div class="grid-header">{REF_COLS[1]}</div>', unsafe_allow_html=True)
+    c[3].markdown(f'<div class="grid-header">{REF_COLS[2]}</div>', unsafe_allow_html=True)
+    c[4].markdown('<div class="grid-header">actions</div>', unsafe_allow_html=True)
+
+# -----------------------------
+# Grid rows (built-in components)
+# -----------------------------
+for idx, fname in enumerate(filenames_in_order):
+    row = df.iloc[idx]
+    col_widths = [6, 2, 2, 2, 3]
+    c = st.columns(col_widths, vertical_alignment="center")
+
+    # Filename
+    with c[0]:
+        st.markdown(f'<div class="fname">{fname}</div>', unsafe_allow_html=True)
+
+    # For each ref column, draw a link button with a colored dot
+    for j, ref in enumerate(REF_COLS, start=1):
+        status = int(row.get(ref, -1))
+        dot = DOTS.get(status, DOTS[-1])
+        url = f"{FILEBROWSER_BASE}/{quote(str(fname))}_fixed-s1_{ref}.MP4"
+        with c[j]:
+            st.link_button(dot, url, use_container_width=True)
+
+    # Actions (only show if resolved != 1)
+    if int(row.get("isresolved", 0)) != 1:
+        with c[4]:
+            aa, bb = st.columns(2, vertical_alignment="center")
+            if aa.button("resolve", key=f"{fname}-resolve"):
+                resolve_record(fname)
+            if bb.button("delete", key=f"{fname}-delete"):
+                delete_record(fname)
+    else:
+        with c[4]:
+            st.markdown('<span style="opacity:0.6">Resolved</span>', unsafe_allow_html=True)
+
+# -----------------------------
+# Optional: Raw data view
+# -----------------------------
+with st.expander("Raw data"):
+    st.dataframe(df[["filename"] + present_refs], use_container_width=True)

streamlit/dockerfile

@@ -0,0 +1,45 @@
+# ---- base ----
+FROM python:3.12-slim
+
+# Prevents Python from writing .pyc files & enables unbuffered logs
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+# System deps (curl used by healthcheck)
+RUN apt-get update && apt-get install -y --no-install-recommends curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user
+RUN useradd -m -u 10001 appuser
+WORKDIR /app
+
+# Copy and install Python deps first (better layer caching)
+# Create requirements.txt with:
+#   streamlit
+#   pandas
+#   sqlalchemy
+#   psycopg2-binary
+#   altair
+COPY requirements.txt .
+RUN python -m pip install --upgrade pip && pip install -r requirements.txt
+
+# Copy app source
+COPY app.py .
+
+# Switch to non-root
+USER appuser
+
+# Streamlit defaults
+ENV STREAMLIT_SERVER_HEADLESS=true \
+    STREAMLIT_BROWSER_GATHER_USAGE_STATS=false
+
+# Expose Streamlit port
+EXPOSE 8501
+
+# Healthcheck: Streamlit readiness
+HEALTHCHECK --interval=30s --timeout=5s --retries=5 \
+  CMD curl -fsS http://localhost:8501/_stcore/health || exit 1
+
+# Run the app
+CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"]

streamlit/requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+sqlalchemy
+psycopg2-binary
+altair