From 4fc19e8c5d97c00cbb1a6b89ed9bd5b1e9a0bcbe Mon Sep 17 00:00:00 2001 From: geezo Date: Mon, 15 Sep 2025 19:42:18 +0000 Subject: [PATCH] commit files --- decorrupt_files.py | 112 ++++++++++++++++++++ docker-compose.yaml | 83 +++++++++++++++ docker_command.sh | 2 + dockerfile | 22 ++++ dockge/data/db-config.json | 3 + dockge/data/dockge.db | Bin 0 -> 40960 bytes dockge/docker-compose.yml | 22 ++++ move_files.py | 9 ++ prime_database.py | 33 ++++++ requirements.txt | 1 + streamlit/app.py | 202 +++++++++++++++++++++++++++++++++++++ streamlit/dockerfile | 45 +++++++++ streamlit/requirements.txt | 5 + 13 files changed, 539 insertions(+) create mode 100755 decorrupt_files.py create mode 100755 docker-compose.yaml create mode 100755 docker_command.sh create mode 100755 dockerfile create mode 100644 dockge/data/db-config.json create mode 100644 dockge/data/dockge.db create mode 100644 dockge/docker-compose.yml create mode 100644 move_files.py create mode 100755 prime_database.py create mode 100755 requirements.txt create mode 100644 streamlit/app.py create mode 100644 streamlit/dockerfile create mode 100644 streamlit/requirements.txt diff --git a/decorrupt_files.py b/decorrupt_files.py new file mode 100755 index 0000000..d29ef65 --- /dev/null +++ b/decorrupt_files.py @@ -0,0 +1,112 @@ + + +import psycopg2 +import subprocess +import os +import shlex + +def get_conn(): + + conn = psycopg2.connect( + host="postgres", + database="mydatabase", + user="myuser", + password="mypassword", + port="5432" # Default PostgreSQL port + ) + + + return conn + +def process_file(file): + + filename = file[0] + ref_x_status = file[1] + ref_12_status = file[2] + ref_16_status = file[3] + ## process ref_x + if ref_x_status < 2: + process_ref(filename, "ref_x.MP4") + ## process ref_12 + if ref_12_status < 2: + process_ref(filename, "ref_12.MP4") + # ### process ref_16 + # if ref_16_status < 2: + # process_ref(filename, "ref_16.MP4") + + +def update_table(filename, ref, status): + + conn = get_conn() + + try: + cursor = conn.cursor() + cursor.execute(f"UPDATE public.decorrupt SET {ref} = {status} WHERE filename = '{filename}'") + conn.commit() + except psycopg2.Error as e: + print(f"Error connecting to or querying the database: {e}") + finally: + # 6. Close Cursor and Connection + if cursor: + cursor.close() + if conn: + conn.close() + + +def process_ref(filename, ref): + print(f"processing file:{filename} ref:{ref}") + + #set to processing + update_table(filename, ref.replace(".MP4", ""), 1) + + # execute docker commnad + execute_docker(filename, ref) + + # rename file + os.rename(f"/motherload/scan/{filename}_fixed-s1.MP4", f"/motherload/scan/{filename}_fixed-s1_{ref}") + + #set to processed + update_table(filename, ref.replace(".MP4", ""), 2) + + +def execute_docker(filename, ref): + motherload_path = "/home/geezo/01_projects/decorrupt/motherload" + command = ["docker", "run", "--rm", "-v", f"{motherload_path}:/motherload", "untrunc", "-s", f"/motherload/reference/{ref}", f"/motherload/scan/{filename}" + + ] + + try: + result = subprocess.run(command, + capture_output=True, text=True, check=True) + print(result.stdout) + except subprocess.CalledProcessError as e: + print(f"Error executing Docker command: {e}") + print(f"Stderr: {e.stderr}") + + + print(f"executing docker... {command}") + + +def get_files(conn): + try: + cursor = conn.cursor() + cursor.execute(f"SELECT * FROM public.decorrupt WHERE isresolved = 0 and isdeleted = 0 ORDER BY filename") + rows = cursor.fetchall() + except psycopg2.Error as e: + print(f"Error connecting to or querying the database: {e}") + finally: + # 6. Close Cursor and Connection + if cursor: + cursor.close() + if conn: + conn.close() + return rows + + +def main(): + files = get_files(get_conn()) + for file in files: + process_file(file) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100755 index 0000000..eec4a86 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,83 @@ + +services: + orchestrator: + build: + context: . + dockerfile: Dockerfile + image: orchestrator:latest + container_name: decorrupt_orchestrator + restart: always + privileged: true + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL: postgres://myuser:mypassword@postgres:5432/mydatabase + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./motherload:/motherload + + postgres: + image: postgres:15 + container_name: postgres_db + restart: always + environment: + POSTGRES_USER: myuser + POSTGRES_PASSWORD: mypassword + POSTGRES_DB: mydatabase + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U myuser -d mydatabase"] + interval: 5s + timeout: 5s + retries: 10 + + + pgadmin: + image: dpage/pgadmin4:latest + container_name: pgadmin + restart: always + environment: + PGADMIN_DEFAULT_EMAIL: admin@example.com + PGADMIN_DEFAULT_PASSWORD: admin + ports: + - "8080:80" + depends_on: + - postgres + volumes: + - pgadmin_data:/var/lib/pgadmin + + streamlit: + image: decorrupt_viz:latest # <-- your built/pushed streamlit image + container_name: streamlit_app + restart: always + depends_on: + postgres: + condition: service_healthy + environment: + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: mydatabase + DB_USER: myuser + DB_PASSWORD: mypassword + TABLE_NAME: "public.decorrupt" # change to your actual table + ports: + - "8501:8501" + + filebrowser: + image: filebrowser/filebrowser:latest + container_name: filebrowser + restart: unless-stopped + ports: + - "8081:80" # access at http://localhost:8081 + volumes: + - ./motherload/scan:/srv # folder you want to browse + - ./filebrowser.db:/database.db + - ./filebrowser.json:/config.json + +volumes: + postgres_data: + pgadmin_data: diff --git a/docker_command.sh b/docker_command.sh new file mode 100755 index 0000000..e44d36a --- /dev/null +++ b/docker_command.sh @@ -0,0 +1,2 @@ +docker run --user 1000 -v ./motherload/playground:/vids -v ./motherload/reference:/refs untrunc -s /refs/ref_x.MP4 /vids/993065877504.mp4 + diff --git a/dockerfile b/dockerfile new file mode 100755 index 0000000..6de2faa --- /dev/null +++ b/dockerfile @@ -0,0 +1,22 @@ +# Use an official lightweight Python image +FROM python:3-alpine + +# Set the working directory in the container +WORKDIR /app + +# Copy requirements file into the container +COPY requirements.txt . + +RUN apk add --no-cache \ + postgresql-libs \ + docker-cli \ + && apk add --no-cache --virtual .build-deps \ + gcc musl-dev postgresql-dev \ + && python3 -m pip install --no-cache-dir -r requirements.txt \ + && apk del .build-deps + +# Copy the rest of your project files into the container +COPY decorrupt_files.py main.py + +# Run your Python file (replace main.py with your script) +CMD ["python", "main.py"] diff --git a/dockge/data/db-config.json b/dockge/data/db-config.json new file mode 100644 index 0000000..6d3ac49 --- /dev/null +++ b/dockge/data/db-config.json @@ -0,0 +1,3 @@ +{ + "type": "sqlite" +} \ No newline at end of file diff --git a/dockge/data/dockge.db b/dockge/data/dockge.db new file mode 100644 index 0000000000000000000000000000000000000000..b7e9e1533d7ca23107c9217837f415e1d8628093 GIT binary patch literal 40960 zcmeI*O>g2x7zgk%4k6IZdZjAzA&0f9NQsqYV@POHsg*YALYmDBY+jn(N)>&v`v?8t4_8aUM=%Ke(d+xO#pgs3o_0&TTd)Uj)7(;-NRafevMSlx~Ju{y1{2trV zj1lZS-mnIe@b2JH?4-q{nDS~JU1HK zCCc*czivMo|9ds}Od_tM(h*2DA@t}uUsSdNqtW#dYf#dKF*q!RN zLs~NHRi;fRru9-JqNXOKrs5-tX7lZ~14ywsHqdcU>6WF)@hRKg&@Vq4ji?h7($}B+ zo}z$wM|w5e`@KD8yP+F3277Vpf5OCzF{?%|9J?zbiIky>j5DWX<~_ + header {visibility: hidden;} + #MainMenu {visibility: hidden;} + footer {visibility: hidden;} + .block-container {padding-top: 1rem; padding-bottom: 2rem;} + .grid-header {font-weight: 600; opacity: 0.8; padding: .25rem 0;} + .fname {white-space: nowrap; overflow: hidden; text-overflow: ellipsis;} + + """, + unsafe_allow_html=True, +) + +# DB config (fill via st.secrets or env) +DB_HOST = "postgres" +DB_PORT = "5432" +DB_NAME = "mydatabase" +DB_USER = "myuser" +DB_PASSWORD = "mypassword" +TABLE_NAME = "decorrupt" +# Optional: restrict columns; leave empty to auto-detect +REF_PREFIX = "ref_" + +# File Browser base (adjust to your host/IP) +FILEBROWSER_BASE = "http://192.168.1.229:8081/files" + +# Expected ref columns (fixed order) +REF_COLS = ["ref_x", "ref_12", "ref_16"] + +# Dot icons for statuses (built-in emoji; no custom CSS/JS) +DOTS = { + -1: "🔴", # unknown/error + 0: "⚪", # not processing (gray/white) + 1: "🟠", # processing (orange) + 2: "🟢", # processed (green) +} + +LABELS = {0: "not processing", 1: "processing", 2: "processed", -1: "unknown"} + + +def update_status(filename: str, column_name: str, new_value: int) -> bool: + """ + Update a single column for a given filename in the specified table. + + Args: + filename (str): The filename to update. + table_name (str): The table where the row exists. + column_name (str): The column to update (e.g., 'ref_x'). + new_value (int): The new status value (e.g., 0, 1, 2). + + Returns: + bool: True if update succeeded, False otherwise. + """ + sql = text(f'UPDATE "decorrupt" SET "{column_name}" = :new_value WHERE filename = :filename') + + try: + engine = get_engine() # reuse your existing get_engine() + with engine.begin() as conn: # auto-commits or rolls back + result = conn.execute(sql, {"new_value": new_value, "filename": filename}) + return result.rowcount > 0 + except Exception as e: + st.error(f"Failed to update {filename}.{column_name}: {e}") + return False +# ----------------------------- +# DB helpers +# ----------------------------- +def get_engine(): + url = f"postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}" + return create_engine(url, pool_pre_ping=True) + + +def load_data(table_name: str) -> pd.DataFrame: + query = text(f'SELECT * FROM "{table_name}" WHERE isdeleted = 0 ORDER BY filename') + with get_engine().connect() as conn: + return pd.read_sql(query, conn) + +# ----------------------------- +# Actions (function placeholders) +# ----------------------------- +def resolve_record(filename: str): + update_status(filename = filename, column_name = "isresolved", new_value = 1) + st.toast(f"{filename} resolved") + + +def delete_record(filename: str): + update_status(filename = filename, column_name = "isdeleted", new_value = 1) + st.toast(f"{filename} deleted") + + +# ----------------------------- +# Load & validate +# ----------------------------- +st.title("File Processing Status") +st.caption("0 = not processing · 1 = processing · 2 = processed") + +if "filename', unsafe_allow_html=True) + c[1].markdown(f'
{REF_COLS[0]}
', unsafe_allow_html=True) + c[2].markdown(f'
{REF_COLS[1]}
', unsafe_allow_html=True) + c[3].markdown(f'
{REF_COLS[2]}
', unsafe_allow_html=True) + c[4].markdown('
actions
', unsafe_allow_html=True) + +# ----------------------------- +# Grid rows (built-in components) +# ----------------------------- +for idx, fname in enumerate(filenames_in_order): + row = df.iloc[idx] + col_widths = [6, 2, 2, 2, 3] + c = st.columns(col_widths, vertical_alignment="center") + + # Filename + with c[0]: + st.markdown(f'
{fname}
', unsafe_allow_html=True) + + # For each ref column, draw a link button with a colored dot + for j, ref in enumerate(REF_COLS, start=1): + status = int(row.get(ref, -1)) + dot = DOTS.get(status, DOTS[-1]) + url = f"{FILEBROWSER_BASE}/{quote(str(fname))}_fixed-s1_{ref}.MP4" + with c[j]: + st.link_button(dot, url, use_container_width=True) + + # Actions (only show if resolved != 1) + if int(row.get("isresolved", 0)) != 1: + with c[4]: + aa, bb = st.columns(2, vertical_alignment="center") + if aa.button("resolve", key=f"{fname}-resolve"): + resolve_record(fname) + if bb.button("delete", key=f"{fname}-delete"): + delete_record(fname) + else: + with c[4]: + st.markdown('Resolved', unsafe_allow_html=True) + +# ----------------------------- +# Optional: Raw data view +# ----------------------------- +with st.expander("Raw data"): + st.dataframe(df[["filename"] + present_refs], use_container_width=True) diff --git a/streamlit/dockerfile b/streamlit/dockerfile new file mode 100644 index 0000000..bb9c66b --- /dev/null +++ b/streamlit/dockerfile @@ -0,0 +1,45 @@ +# ---- base ---- +FROM python:3.12-slim + +# Prevents Python from writing .pyc files & enables unbuffered logs +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +# System deps (curl used by healthcheck) +RUN apt-get update && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +# Create non-root user +RUN useradd -m -u 10001 appuser +WORKDIR /app + +# Copy and install Python deps first (better layer caching) +# Create requirements.txt with: +# streamlit +# pandas +# sqlalchemy +# psycopg2-binary +# altair +COPY requirements.txt . +RUN python -m pip install --upgrade pip && pip install -r requirements.txt + +# Copy app source +COPY app.py . + +# Switch to non-root +USER appuser + +# Streamlit defaults +ENV STREAMLIT_SERVER_HEADLESS=true \ + STREAMLIT_BROWSER_GATHER_USAGE_STATS=false + +# Expose Streamlit port +EXPOSE 8501 + +# Healthcheck: Streamlit readiness +HEALTHCHECK --interval=30s --timeout=5s --retries=5 \ + CMD curl -fsS http://localhost:8501/_stcore/health || exit 1 + +# Run the app +CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"] diff --git a/streamlit/requirements.txt b/streamlit/requirements.txt new file mode 100644 index 0000000..598529d --- /dev/null +++ b/streamlit/requirements.txt @@ -0,0 +1,5 @@ +streamlit +pandas +sqlalchemy +psycopg2-binary +altair