From 6bdce68f3f223c6e01ff0831549d324bc7f78c22 Mon Sep 17 00:00:00 2001 From: KRZYSZTOF RUDNICKI Date: Wed, 16 Oct 2024 12:12:13 +0200 Subject: [PATCH 01/15] feat: bash scripts --- process_table.sh | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ upgrade.sh | 4 ++++ 2 files changed, 57 insertions(+) create mode 100755 process_table.sh create mode 100755 upgrade.sh diff --git a/process_table.sh b/process_table.sh new file mode 100755 index 0000000..553721b --- /dev/null +++ b/process_table.sh @@ -0,0 +1,53 @@ + +#!/bin/bash + +process_table_schema() { + while IFS=$'\t' read -r column_name _ data_type _; do + # Print the column name and data type + echo -e "$column_name\t$data_type" + done < "$1" +} + +input_file="$1" + +# Check if a file is provided as an argument +if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Process the provided file and skip the first row +first_line=true +process_table_schema "$input_file" | while IFS=$'\t' read -r column_name data_type; do + if [ "$first_line" = true ]; then + first_line=false + continue + fi + case "$data_type" in + "timestamp") + sqlalchemy_type="DateTime" + ;; + "int"|"integer"|"int4") + sqlalchemy_type="Integer" + ;; + "varchar"*|"text") + sqlalchemy_type="String" # handles types like varchar(256) + ;; + "boolean"|"bool") + sqlalchemy_type="Boolean" + ;; + "float"|"float8") + sqlalchemy_type="Float" + ;; + "serial4") + sqlalchemy_type="Integer" + ;; + "numeric"*) + sqlalchemy_type="Numeric" # handles types like numeric(12, 2) + ;; + *) + sqlalchemy_type="UNDEFINED_CHANGE_ME" # default to UNDEFINED_CHANGE_ME if data type is unrecognized + ;; + esac + echo "$column_name = Column($sqlalchemy_type)" +done \ No newline at end of file diff --git a/upgrade.sh b/upgrade.sh new file mode 100755 index 0000000..911ddc2 --- /dev/null +++ b/upgrade.sh @@ -0,0 +1,4 @@ +#!/bin/sh +apt-get -y update && apt-get -y upgrade && apt -y dist-upgrade +apt -y autoremove +aptitude -y update && aptitude -y safe-upgrade && aptitude -y dist-upgrade From 6586e82c4599487706c2312214c5b9420510f96c Mon Sep 17 00:00:00 2001 From: Krzysztof Rudnicki Date: Thu, 14 Nov 2024 20:59:15 +0100 Subject: [PATCH 02/15] feat: download script --- .gitignore | 7 +++++++ download.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 .gitignore create mode 100755 download.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5b6899e --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.txt +*.webm* +*.mp4* +*.mp3* +*.ogg* +*.wav* +*.m4a* \ No newline at end of file diff --git a/download.sh b/download.sh new file mode 100755 index 0000000..8543cb6 --- /dev/null +++ b/download.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Check if there are any .txt files in the current directory +txt_files=(*.txt) +if [ ${#txt_files[@]} -eq 0 ]; then + echo "No .txt files found in the current directory!" + exit 1 +fi + +total_files=0 +total_size=0 +downloaded_files=0 +downloaded_size=0 + +# Calculate total number of files and total size to download +for file in *.txt; do + while IFS= read -r url; do + if [[ -n "$url" ]]; then + total_files=$((total_files + 1)) + size=$(wget --spider "$url" 2>&1 | grep Length | awk '{print $2}') + total_size=$((total_size + size)) + fi + done < "$file" +done + +# Loop through each .txt file and download each URL in parallel +for file in *.txt; do + echo "Processing $file..." + while IFS= read -r url; do + if [[ -n "$url" ]]; then + { + wget -q --show-progress "$url" + downloaded_files=$((downloaded_files + 1)) + size=$(wget --spider "$url" 2>&1 | grep Length | awk '{print $2}') + downloaded_size=$((downloaded_size + size)) + remaining_files=$((total_files - downloaded_files)) + remaining_size=$((total_size - downloaded_size)) + echo "Downloaded: $downloaded_files/$total_files files, $downloaded_size/$total_size bytes" + echo "Remaining: $remaining_files files, $remaining_size bytes" + } & + fi + done < "$file" +done + +# Wait for all background jobs to complete +wait \ No newline at end of file From 166f1d35a9c9ecd62ed70ab4c3f576af4966c6a7 Mon Sep 17 00:00:00 2001 From: Krzysztof Rudnicki Date: Thu, 5 Dec 2024 19:35:32 +0100 Subject: [PATCH 03/15] feat: added script for converting video to mp4 and to keep it below discord video size limit --- convert.sh | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 convert.sh diff --git a/convert.sh b/convert.sh new file mode 100755 index 0000000..c1123e7 --- /dev/null +++ b/convert.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Default values +TARGET_EXT="mp4" +TARGET_SIZE=10M + +# Parse arguments +if [ -n "$1" ]; then + INPUT_PATH="$1" +else + INPUT_PATH="." +fi + +if [ -n "$2" ]; then + TARGET_EXT="$2" +fi + +if [ -n "$3" ]; then + TARGET_SIZE="$3" +fi + +# Create output directory +OUTPUT_DIR="converted" +mkdir -p "$OUTPUT_DIR" + +# Function to convert video +convert_video() { + local input_file="$1" + local output_file="$OUTPUT_DIR/${input_file%.*}.$TARGET_EXT" + + # Get video duration in seconds + DURATION=$(ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$input_file") + echo "Duration: $DURATION seconds" + + # Convert target size to bytes + TARGET_SIZE_BYTES=$(numfmt --from=iec "$TARGET_SIZE") + + # Calculate target bitrate in kilobits per second + TARGET_BITRATE=$(echo "($TARGET_SIZE_BYTES * 8) / $DURATION / 2000" | bc) # Reduce by 10% to ensure size is below target + + # Convert video + ffmpeg -i "$input_file" -b:v "${TARGET_BITRATE}k" -c:a copy "$output_file" + + # Get original and converted video sizes + ORIGINAL_SIZE=$(stat -c%s "$input_file") + CONVERTED_SIZE=$(stat -c%s "$output_file") + + # Print out details + echo "Original size: $(numfmt --to=iec $ORIGINAL_SIZE)" + echo "Video length: $DURATION seconds" + echo "Target size: $TARGET_SIZE" + echo "Converted size: $(numfmt --to=iec $CONVERTED_SIZE)" + echo "Target bitrate: ${TARGET_BITRATE}kbps" +} + +# Export function for find command +export -f convert_video +export TARGET_EXT +export TARGET_SIZE +export OUTPUT_DIR + +# Find and convert videos +if [ -d "$INPUT_PATH" ]; then + find "$INPUT_PATH" -type f -name "*.mp4" -o -name "*.mkv" -o -name "*.avi" -o -name "*.webm" -exec bash -c 'convert_video "$0"' {} \; +else + convert_video "$INPUT_PATH" +fi From 037ca27fd2ad3996e7147d892df66f5eb14ab0b8 Mon Sep 17 00:00:00 2001 From: Krzysztof Rudnicki Date: Thu, 5 Dec 2024 19:36:29 +0100 Subject: [PATCH 04/15] feat: add flags to video conversion --- convert.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert.sh b/convert.sh index c1123e7..7c19dfd 100755 --- a/convert.sh +++ b/convert.sh @@ -39,7 +39,7 @@ convert_video() { TARGET_BITRATE=$(echo "($TARGET_SIZE_BYTES * 8) / $DURATION / 2000" | bc) # Reduce by 10% to ensure size is below target # Convert video - ffmpeg -i "$input_file" -b:v "${TARGET_BITRATE}k" -c:a copy "$output_file" + ffmpeg -i "$input_file" -vcodec libx264 -b:v "${TARGET_BITRATE}k" -preset veryslow -acodec aac -c:a copy "$output_file" # Get original and converted video sizes ORIGINAL_SIZE=$(stat -c%s "$input_file") From f49a7d909b1528f29c11f67dd30498497f9c3bf2 Mon Sep 17 00:00:00 2001 From: Krzysztof Rudnicki Date: Thu, 5 Dec 2024 19:38:33 +0100 Subject: [PATCH 05/15] feat: convert video now fillters videos --- convert.sh | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/convert.sh b/convert.sh index 7c19dfd..3e03cc5 100755 --- a/convert.sh +++ b/convert.sh @@ -36,7 +36,7 @@ convert_video() { TARGET_SIZE_BYTES=$(numfmt --from=iec "$TARGET_SIZE") # Calculate target bitrate in kilobits per second - TARGET_BITRATE=$(echo "($TARGET_SIZE_BYTES * 8) / $DURATION / 2000" | bc) # Reduce by 10% to ensure size is below target + TARGET_BITRATE=$(echo "($TARGET_SIZE_BYTES * 8) / $DURATION / 2000" | bc) # Convert video ffmpeg -i "$input_file" -vcodec libx264 -b:v "${TARGET_BITRATE}k" -preset veryslow -acodec aac -c:a copy "$output_file" @@ -53,15 +53,34 @@ convert_video() { echo "Target bitrate: ${TARGET_BITRATE}kbps" } -# Export function for find command +# Function to move video if already below target size and in desired format +move_video() { + local input_file="$1" + local output_file="$OUTPUT_DIR/${input_file##*/}" + + # Get original video size + ORIGINAL_SIZE=$(stat -c%s "$input_file") + + # Check if video is below target size and in desired format + if [[ "$ORIGINAL_SIZE" -le "$TARGET_SIZE_BYTES" && "${input_file##*.}" == "$TARGET_EXT" ]]; then + mv "$input_file" "$output_file" + echo "Moved $input_file to $output_file" + else + convert_video "$input_file" + fi +} + +# Export functions for find command export -f convert_video +export -f move_video export TARGET_EXT export TARGET_SIZE +export TARGET_SIZE_BYTES export OUTPUT_DIR -# Find and convert videos +# Find and process videos if [ -d "$INPUT_PATH" ]; then - find "$INPUT_PATH" -type f -name "*.mp4" -o -name "*.mkv" -o -name "*.avi" -o -name "*.webm" -exec bash -c 'convert_video "$0"' {} \; + find "$INPUT_PATH" -type f -name "*.mp4" -o -name "*.mkv" -o -name "*.avi" -o -name "*.webm" -exec bash -c 'move_video "$0"' {} \; else - convert_video "$INPUT_PATH" + move_video "$INPUT_PATH" fi From 2fb23a9f3381b82d8a1a5bde7fd716ae362c8fd5 Mon Sep 17 00:00:00 2001 From: Krzysztof Rudnicki Date: Mon, 16 Dec 2024 18:09:26 +0100 Subject: [PATCH 06/15] feat: conveert mkv videos additionally --- convert.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert.sh b/convert.sh index 3e03cc5..7c02225 100755 --- a/convert.sh +++ b/convert.sh @@ -80,7 +80,7 @@ export OUTPUT_DIR # Find and process videos if [ -d "$INPUT_PATH" ]; then - find "$INPUT_PATH" -type f -name "*.mp4" -o -name "*.mkv" -o -name "*.avi" -o -name "*.webm" -exec bash -c 'move_video "$0"' {} \; + find "$INPUT_PATH" \( -name "*.mkv" -o -name "*.mp4" -o -name "*.avi" -o -name "*.webm" \) -type f -exec bash -c 'move_video "$0"' {} \; else move_video "$INPUT_PATH" fi From 2d6a7815e810a93fb57b9206be2b70c8c65543bc Mon Sep 17 00:00:00 2001 From: Krzysztof Rudnicki Date: Sun, 5 Jan 2025 15:26:11 +0100 Subject: [PATCH 07/15] feat: added script for comperssing folder --- compress_images.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 compress_images.sh diff --git a/compress_images.sh b/compress_images.sh new file mode 100644 index 0000000..07eb54a --- /dev/null +++ b/compress_images.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Directory containing the images +directory="./images" + +# Compression level (default to 0 if not provided) +compression_level=${1:-0} + +# Create output directory, overwrite if it already exists +output_directory="${directory}/webp" +rm -rf "$output_directory" +mkdir -p "$output_directory" + +# Iterate through each file in the directory +for file in "$directory"/*.{jpg,jpeg,png,bmp,tiff}; do + # Skip if no matching files are found + [ -e "$file" ] || continue + + # Extract the filename without extension + filename=$(basename "$file") + filename_no_ext="${filename%.*}" + + # Convert the file to WebP with specified compression level + cwebp -q "$compression_level" "$file" -o "$output_directory/${filename_no_ext}.webp" + + echo "Converted: $file -> $output_directory/${filename_no_ext}.webp" +done + +echo "All images have been converted to WebP with compression level $compression_level." From ab90468fd03ad2528b12b804dad730b74fbb09a0 Mon Sep 17 00:00:00 2001 From: KRZYSZTOF RUDNICKI Date: Tue, 7 Jan 2025 13:35:25 +0100 Subject: [PATCH 08/15] feat: add script for generating random subfolders structures --- .gitignore | 3 +- generate_subfolders.sh | 73 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100755 generate_subfolders.sh diff --git a/.gitignore b/.gitignore index 5b6899e..4750368 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ *.mp3* *.ogg* *.wav* -*.m4a* \ No newline at end of file +*.m4a* +main_folder \ No newline at end of file diff --git a/generate_subfolders.sh b/generate_subfolders.sh new file mode 100755 index 0000000..843ee02 --- /dev/null +++ b/generate_subfolders.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Function to generate random number between two values +random_number() { + echo $((RANDOM % ($2 - $1 + 1) + $1)) +} + +# Function to generate random string with non-computer-friendly characters +random_string() { + local length=$1 + tr -dc 'a-zA-Z0-9!@#$%^&*()_+{}|:<>?~' < /dev/urandom | head -c $length +} + +# Function to calculate total number of folders to be created +calculate_total_folders() { + local depth=$1 + local total=0 + if [ "$depth" -le 10 ]; then + local num_subfolders=$(random_number 1 50) + total=$((num_subfolders + total)) + for ((i=1; i<=num_subfolders; i++)); do + total=$((total + $(calculate_total_folders $((depth + 1))))) + done + fi + echo $total +} + +# Function to create folders and files recursively +create_structure() { + local current_depth=$1 + local parent_dir=$2 + local start_time=$3 + + if [ "$current_depth" -le 10 ]; then + local num_subfolders=$(random_number 1 50) + echo "Creating $num_subfolders subfolders at depth $current_depth" + for ((i=1; i<=num_subfolders; i++)); do + local subfolder="$parent_dir/$(random_string 255)" + mkdir -p "$subfolder" + ((generated_folders++)) + + # Display progress + local elapsed_time=$(( $(date +%s) - start_time )) + local estimated_total_time=$(( elapsed_time * total_folders / generated_folders )) + local remaining_time=$(( estimated_total_time - elapsed_time )) + echo "Generated: $generated_folders/$total_folders folders. Estimated time left: $remaining_time seconds." + + # Create random number of empty files + local num_files=$(random_number 10 100) + echo "Creating $num_files files" + for ((j=1; j<=num_files; j++)); do + touch "$subfolder/$(random_string 255)" + done + + # Recursively create subfolders + create_structure $((current_depth + 1)) "$subfolder" $start_time + done + fi +} + +# Main folder +main_folder="/home/k.rudnicki@aiclearing.com/testsAndMisc/Bash/main_folder" +mkdir -p "$main_folder" + +# Calculate total folders to be created +# total_folders=$(calculate_total_folders 1) +generated_folders=0 + +echo "Total folders to be generated: $total_folders" + +# Start creating structure from the main folder +start_time=$(date +%s) +create_structure 1 "$main_folder" $start_time \ No newline at end of file From bfedea88a61cfae24480144f04ba7e76794335f3 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 26 Jan 2025 12:03:44 +0100 Subject: [PATCH 09/15] feat: script adding/subtracting randomly from every number in an array --- copyFolder.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 copyFolder.sh diff --git a/copyFolder.sh b/copyFolder.sh new file mode 100755 index 0000000..d6bc845 --- /dev/null +++ b/copyFolder.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Get the list of directories in the current script directory +directories=($(find . -maxdepth 1 -type d ! -name .)) + +# Check if there is exactly one directory +if [ ${#directories[@]} -ne 1 ]; then + echo "Error: There should be exactly one folder in the current directory." + exit 1 +fi + +# Get the name of the single directory +folder_name=${directories[0]} + +random_string() { + local length=$1 + tr -dc 'a-zA-Z0-9!@#$%^&*()_+{}|:<>?~' < /dev/urandom | head -c $length +} + +# Number of copies to create (default 100) +num_copies=${1:-100} + +# Create the specified number of copies +for ((i=1; i<=num_copies; i++)); do + new_folder_name="$(random_string 255)" + cp -r "$folder_name" "$new_folder_name" + echo "Folder copied and renamed to '$new_folder_name'" +done \ No newline at end of file From ecba8a4bde64b097e2c5327a7749e918ea181105 Mon Sep 17 00:00:00 2001 From: KRZYSZTOF RUDNICKI Date: Tue, 16 Sep 2025 13:40:58 +0200 Subject: [PATCH 10/15] feat: libre translate setup script --- libre_translate.sh | 381 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100755 libre_translate.sh diff --git a/libre_translate.sh b/libre_translate.sh new file mode 100755 index 0000000..502eef6 --- /dev/null +++ b/libre_translate.sh @@ -0,0 +1,381 @@ +#!/usr/bin/env bash +set -euo pipefail + +# LibreTranslate full setup script (Docker-based) +# Features: +# - Installs Docker if missing (optional --no-docker-install) +# - Pulls libretranslate image (tag configurable) +# - Creates persistent data + cache directories +# - Optionally pre-downloads language models +# - Generates or accepts an API key; can disable auth +# - (Removed) systemd service setup – now always ephemeral +# - Health check + sample translation +# - Uninstall mode removes container, image, service, and data (optional keep data) +# - Idempotent: safe to re-run for upgrades (will pull newer image) + +SCRIPT_NAME=$(basename "$0") +VERSION="1.0.0" + +# Defaults +IMAGE="libretranslate/libretranslate" +TAG="latest" +SERVICE_NAME="libretranslate" +DOCKER_INSTALL=1 +# Systemd removed – always run ephemeral container +API_KEY="" +GENERATE_API_KEY=1 +DISABLE_API_KEY=0 +PORT=5000 +HOST=0.0.0.0 +DATA_DIR="/var/lib/libretranslate" +CACHE_DIR="${DATA_DIR}/cache" +CONFIG_DIR="/etc/libretranslate" +ENV_FILE="${CONFIG_DIR}/libretranslate.env" +PULL_ONLY=0 +PRELOAD_LANGS="" +UNINSTALL=0 +KEEP_DATA=0 +HEALTH_TIMEOUT=15 +EXTRA_ENV=() +NO_COLOR=0 +KEEP_ALIVE=0 +RUN_COMMAND=() +DEBUG=0 + +# Colors +if [[ -t 1 && ${NO_COLOR} -eq 0 ]]; then + GREEN="\e[32m"; YELLOW="\e[33m"; RED="\e[31m"; BLUE="\e[34m"; BOLD="\e[1m"; RESET="\e[0m" +else + GREEN=""; YELLOW=""; RED=""; BLUE=""; BOLD=""; RESET="" +fi + +log() { echo -e "${BLUE}[INFO]${RESET} $*"; } +warn() { echo -e "${YELLOW}[WARN]${RESET} $*" >&2; } +err() { echo -e "${RED}[ERR ]${RESET} $*" >&2; } +success() { echo -e "${GREEN}[OK ]${RESET} $*"; } + +usage() { + cat </dev/null 2>&1; then + key=$(openssl rand -base64 48 | tr -dc 'A-Za-z0-9' | head -c 40 || true) + fi + fi + if [[ -z $key || ${#key} -lt 20 ]]; then + # Last resort static warning key (should not happen) + key="LT$(date +%s)$$RANDOM" + fi + printf '%s' "$key" +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { err "Required command '$1' not found"; return 1; } +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --image) IMAGE="$2"; shift 2;; + --tag) TAG="$2"; shift 2;; + --port) PORT="$2"; shift 2;; + --host) HOST="$2"; shift 2;; + --data-dir) DATA_DIR="$2"; CACHE_DIR="${DATA_DIR}/cache"; shift 2;; + --cache-dir) CACHE_DIR="$2"; shift 2;; + --no-docker-install) DOCKER_INSTALL=0; shift;; + --keep-alive) KEEP_ALIVE=1; shift;; + --) shift; RUN_COMMAND=("$@"); break;; + --api-key) API_KEY="$2"; GENERATE_API_KEY=0; shift 2;; + --generate-api-key) GENERATE_API_KEY=1; shift;; + --disable-api-key) DISABLE_API_KEY=1; shift;; + --preload-langs) PRELOAD_LANGS="$2"; shift 2;; + --env) EXTRA_ENV+=("$2"); shift 2;; + --pull-only) PULL_ONLY=1; shift;; + --uninstall) UNINSTALL=1; shift;; + --purge) UNINSTALL=1; KEEP_DATA=0; shift;; + --keep-data) KEEP_DATA=1; shift;; + --health-timeout) HEALTH_TIMEOUT="$2"; shift 2;; + --no-color) NO_COLOR=1; shift;; + --debug) DEBUG=1; shift;; + -h|--help) usage; exit 0;; + -v|--version) echo "${VERSION}"; exit 0;; + *) err "Unknown argument: $1"; usage; exit 1;; + esac + done +} + +ensure_root() { + if [[ $EUID -ne 0 ]]; then + err "This script must run as root (or via sudo)."; exit 1 + fi +} + +install_docker() { + if command -v docker >/dev/null 2>&1; then + log "Docker already installed" + return 0 + fi + if [[ ${DOCKER_INSTALL} -eq 0 ]]; then + err "Docker is not installed and --no-docker-install specified."; exit 1 + fi + log "Installing Docker..." + if command -v apt-get >/dev/null 2>&1; then + apt-get update -y + apt-get install -y ca-certificates curl gnupg + install -d -m 0755 /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + echo \ +"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(. /etc/os-release; echo "$VERSION_CODENAME") stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update -y + apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + else + err "Unsupported package manager. Please install Docker manually."; exit 1 + fi + # Attempt to start docker daemon if dockerd exists and systemctl available; otherwise rely on user + if command -v systemctl >/dev/null 2>&1; then + (systemctl enable --now docker 2>/dev/null && success "Docker installed and started") || warn "Docker installed; ensure dockerd is running" + else + warn "Docker installed; please ensure docker daemon is running" + fi +} + +pull_image() { + log "Pulling image ${IMAGE}:${TAG}" + docker pull "${IMAGE}:${TAG}" + success "Image pulled" +} + +detect_container_user() { + # Determine uid/gid of configured user inside image so host dirs can be chowned + if ! command -v docker >/dev/null 2>&1; then + return 0 + fi + local uid gid + uid=$(docker run --rm --entrypoint /usr/bin/id "${IMAGE}:${TAG}" -u 2>/dev/null || echo "") + gid=$(docker run --rm --entrypoint /usr/bin/id "${IMAGE}:${TAG}" -g 2>/dev/null || echo "") + if [[ -n $uid && -n $gid ]]; then + CONTAINER_UID=$uid + CONTAINER_GID=$gid + fi +} + +write_env_file() { + mkdir -p "${CONFIG_DIR}" "${DATA_DIR}" "${CACHE_DIR}" + detect_container_user + if [[ -n ${CONTAINER_UID:-} && -n ${CONTAINER_GID:-} ]]; then + if command -v stat >/dev/null 2>&1; then + for d in "${DATA_DIR}" "${CACHE_DIR}"; do + if [[ -d $d ]]; then + CUR_UID=$(stat -c %u "$d" 2>/dev/null || echo -1) + if [[ ${CUR_UID} -ne ${CONTAINER_UID} ]]; then + chown ${CONTAINER_UID}:${CONTAINER_GID} "$d" 2>/dev/null || warn "Unable to chown $d to ${CONTAINER_UID}:${CONTAINER_GID}" + fi + fi + done + fi + fi + if [[ ${DISABLE_API_KEY} -eq 1 ]]; then + API_KEY_LINE="LT_NO_API_KEY=true" + else + if [[ -z ${API_KEY} && ${GENERATE_API_KEY} -eq 1 ]]; then + API_KEY=$(gen_api_key) + GENERATED=1 + else + GENERATED=0 + fi + API_KEY_LINE="LT_API_KEYS=${API_KEY}" + fi + + { echo "# LibreTranslate environment file"; echo "# Generated $(date -u +%Y-%m-%dT%H:%M:%SZ)"; echo "${API_KEY_LINE}"; + [[ -n ${PRELOAD_LANGS} ]] && echo "LT_PRELOAD_LANGS=${PRELOAD_LANGS}"; + for kv in "${EXTRA_ENV[@]:-}"; do echo "$kv"; done; } > "${ENV_FILE}.tmp" + mv "${ENV_FILE}.tmp" "${ENV_FILE}" + chmod 600 "${ENV_FILE}" + success "Environment file written: ${ENV_FILE}" +} + +start_container_ephemeral() { + log "Starting ephemeral container..." + docker rm -f "${SERVICE_NAME}" >/dev/null 2>&1 || true + docker run -d --name "${SERVICE_NAME}" \ + --env-file "${ENV_FILE}" \ + -v "${DATA_DIR}:/home/libretranslate/.local/share/argos-translate" \ + -v "${CACHE_DIR}:/app/cache" \ + -p "${PORT}:${PORT}" \ + "${IMAGE}:${TAG}" \ + --host 0.0.0.0 --port ${PORT} + success "Container started (ephemeral)" + echo + echo "Endpoint (pending readiness): http://$(hostname -I | awk '{print $1}'):${PORT}" + echo "Waiting for health..." +} + +health_check() { + local start=$(date +%s) + local url="http://127.0.0.1:${PORT}/languages" + local attempt=0 + while true; do + attempt=$((attempt+1)) + if curl ${DEBUG:+-v} -fsS "$url" >/dev/null 2>&1; then + success "Service healthy (attempt $attempt)" + return 0 + else + [[ $DEBUG -eq 1 ]] && log "Health attempt $attempt failed" + fi + if (( $(date +%s) - start > HEALTH_TIMEOUT )); then + err "Health check failed after ${HEALTH_TIMEOUT}s (attempts: $attempt)" + docker logs --tail 200 "${SERVICE_NAME}" || true + return 1 + fi + sleep 0.5 + done +} + +sample_request() { + if [[ ${DISABLE_API_KEY} -eq 0 ]]; then + local key="${API_KEY}" + else + local key="" + fi + log "Performing sample translation (en->es)..." + local DATA='{"q":"Hello world","source":"en","target":"es","format":"text"}' + if [[ -n $key ]]; then + curl -fsS -H "Content-Type: application/json" -H "Authorization: ${key}" -d "$DATA" "http://127.0.0.1:${PORT}/translate" || warn "Sample request failed" + else + curl -fsS -H "Content-Type: application/json" -d "$DATA" "http://127.0.0.1:${PORT}/translate" || warn "Sample request failed" + fi + echo +} + +uninstall_all() { + log "Uninstalling LibreTranslate (ephemeral mode)..." + docker rm -f "${SERVICE_NAME}" 2>/dev/null || true + docker rmi "${IMAGE}:${TAG}" 2>/dev/null || true + if [[ ${KEEP_DATA} -eq 0 ]]; then + rm -rf "${DATA_DIR}" "${CONFIG_DIR}" || true + success "Data directories removed" + else + log "Data kept in ${DATA_DIR} and ${CONFIG_DIR}" + fi + success "Uninstall complete" + exit 0 +} + +main() { + parse_args "$@" + ensure_root + + if [[ ${UNINSTALL} -eq 1 ]]; then + uninstall_all + fi + + install_docker + pull_image + if [[ ${PULL_ONLY} -eq 1 ]]; then + log "Pull-only requested, exiting." + exit 0 + fi + + write_env_file + + # Always ephemeral now + start_container_ephemeral + + health_check + sample_request || true + + # If a command is provided, run it and then shutdown container + if [[ ${#RUN_COMMAND[@]} -gt 0 ]]; then + log "Running user command: ${RUN_COMMAND[*]}" + set +e + "${RUN_COMMAND[@]}" + CMD_STATUS=$? + set -e + log "Command exited with status ${CMD_STATUS}; stopping container" + docker stop "${SERVICE_NAME}" >/dev/null 2>&1 || true + exit ${CMD_STATUS} + fi + + if [[ ${KEEP_ALIVE} -eq 1 ]]; then + log "Tailing logs (Ctrl-C to stop and remove container)" + trap 'log "Stopping container"; docker stop "${SERVICE_NAME}" >/dev/null 2>&1 || true; exit 0' INT TERM + docker logs -f "${SERVICE_NAME}" + log "Logs ended; stopping container" + docker stop "${SERVICE_NAME}" >/dev/null 2>&1 || true + else + log "Ephemeral container left running in background (id: $(docker inspect --format '{{.Id}}' ${SERVICE_NAME} 2>/dev/null || echo unknown))" + log "Stop manually with: docker stop ${SERVICE_NAME}" + fi + + echo + echo "${BOLD}LibreTranslate is ready.${RESET}" + echo "Endpoint: http://$(hostname -I | awk '{print $1}'):${PORT}" + if [[ ${DISABLE_API_KEY} -eq 0 ]]; then + if [[ ${GENERATED:-0} -eq 1 ]]; then + echo "Generated API key: ${API_KEY}" + else + echo "API key: ${API_KEY}" + fi + echo "Use header: Authorization: " + else + echo "API key authentication DISABLED (public instance)." + fi + [[ -n ${PRELOAD_LANGS} ]] && echo "Preloaded languages requested: ${PRELOAD_LANGS}" || true + echo "Environment file: ${ENV_FILE}" + echo "Manage: docker logs -f ${SERVICE_NAME} | docker stop ${SERVICE_NAME}" + echo "Uninstall: sudo ${SCRIPT_NAME} --uninstall" + echo +} + +main "$@" + From 68fbd82d78623e9a9b588227faeae44719e67287 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 5 Oct 2025 18:13:44 +0200 Subject: [PATCH 11/15] feat: scirpt for installing unity mcp --- install_unity_mcp.sh | 231 +++++++++++++++++++++++++++++++++++++++++++ mcp_readme.md | 189 +++++++++++++++++++++++++++++++++++ 2 files changed, 420 insertions(+) create mode 100755 install_unity_mcp.sh create mode 100644 mcp_readme.md diff --git a/install_unity_mcp.sh b/install_unity_mcp.sh new file mode 100755 index 0000000..6fd34c7 --- /dev/null +++ b/install_unity_mcp.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_NAME="$(basename "$0")" + +RED="\033[31m" +YELLOW="\033[33m" +BLUE="\033[34m" +RESET="\033[0m" + +info() { + printf "%b[%s]%b %s\n" "$BLUE" "$SCRIPT_NAME" "$RESET" "$*" +} + +warn() { + printf "%b[%s]%b %s\n" "$YELLOW" "$SCRIPT_NAME" "$RESET" "$*" >&2 +} + +error() { + printf "%b[%s]%b %s\n" "$RED" "$SCRIPT_NAME" "$RESET" "$*" >&2 +} + +require_command() { + local cmd="$1" + local package_hint="${2:-}" + + if ! command -v "$cmd" >/dev/null 2>&1; then + if [[ -n "$package_hint" ]]; then + error "Missing command '$cmd'. Try installing the package: $package_hint" + else + error "Missing command '$cmd'." + fi + exit 1 + fi +} + +ensure_pacman_packages() { + local packages=("python" "git" "curl" "jq" "code") + local missing=() + for pkg in "${packages[@]}"; do + if ! pacman -Qi "$pkg" >/dev/null 2>&1; then + missing+=("$pkg") + fi + done + + if (( ${#missing[@]} > 0 )); then + info "Installing required packages with pacman: ${missing[*]}" + sudo pacman -S --needed --noconfirm "${missing[@]}" + else + info "All required pacman packages are already installed." + fi +} + +install_uv() { + if command -v uv >/dev/null 2>&1; then + info "uv is already installed." + return + fi + + info "Installing uv toolchain manager via official installer." + curl -LsSf https://astral.sh/uv/install.sh | sh + + local local_bin="$HOME/.local/bin" + if [[ ":$PATH:" != *":$local_bin:"* ]]; then + warn "Adding $local_bin to PATH in ~/.profile and ~/.zshrc. Open a new shell to apply." + printf '\nexport PATH="$HOME/.local/bin:$PATH"\n' >> "$HOME/.profile" + printf '\nexport PATH="$HOME/.local/bin:$PATH"\n' >> "$HOME/.zshrc" + fi +} + +ensure_unity_hub() { + if command -v unityhub >/dev/null 2>&1; then + info "Unity Hub already installed." + return + fi + + if command -v yay >/dev/null 2>&1; then + info "Installing Unity Hub from AUR using yay." + yay -S --needed --noconfirm unityhub + elif command -v flatpak >/dev/null 2>&1; then + warn "Unity Hub not found. Attempting Flatpak installation." + flatpak install -y com.unity.UnityHub || warn "Flatpak installation failed. Install Unity Hub manually via https://unity.com/download" + else + warn "Unity Hub not found and neither yay nor flatpak is available. Install Unity Hub manually from https://unity.com/download." + fi +} + +sync_unity_mcp_repo() { + local data_home="${XDG_DATA_HOME:-$HOME/.local/share}" + local unity_mcp_root="$data_home/UnityMCP" + local repo_dir="$unity_mcp_root/unity-mcp-repo" + local server_link="$unity_mcp_root/UnityMcpServer" + local candidates=( + "UnityMcpServer" + "UnityMcpBridge/UnityMcpServer" + "UnityMcpBridge/UnityMcpServer~" + ) + local server_subdir="" + + mkdir -p "$unity_mcp_root" + + if [[ -d "$repo_dir/.git" ]]; then + info "Updating existing unity-mcp repository." + git -C "$repo_dir" pull --ff-only + else + info "Cloning unity-mcp repository." + rm -rf "$repo_dir" + git clone --depth=1 https://github.com/CoplayDev/unity-mcp.git "$repo_dir" + fi + + for candidate in "${candidates[@]}"; do + if [[ -d "$repo_dir/$candidate/src" ]]; then + server_subdir="$candidate" + break + fi + done + + if [[ -z "$server_subdir" ]]; then + error "UnityMcpServer src directory not found. Checked candidates: ${candidates[*]}" + error "Repository layout may have changed. Inspect $repo_dir for the new server location." + exit 1 + fi + + ln -sfn "$repo_dir/$server_subdir" "$server_link" + info "UnityMcpServer synchronized at $server_link (source: $server_subdir)" +} + +configure_vscode_mcp() { + local data_home="${XDG_DATA_HOME:-$HOME/.local/share}" + local server_src="$data_home/UnityMCP/UnityMcpServer/src" + local mcp_config_dir="$HOME/.config/Code/User" + local mcp_config="$mcp_config_dir/mcp.json" + local tmp + + if [[ ! -d "$server_src" ]]; then + error "Server source directory $server_src is missing." + exit 1 + fi + + mkdir -p "$mcp_config_dir" + + if [[ ! -f "$mcp_config" ]]; then + info "Creating new VS Code MCP configuration at $mcp_config" + echo '{}' > "$mcp_config" + else + info "Updating existing VS Code MCP configuration at $mcp_config" + fi + + tmp="$(mktemp)" + + if ! jq '.' "$mcp_config" >/dev/null 2>&1; then + error "Existing $mcp_config is not valid JSON. Please fix it before running this script again." + exit 1 + fi + + jq \ + --arg path "$server_src" \ + '(.servers //= {}) | + .servers.unityMCP = { + command: "uv", + args: ["--directory", $path, "run", "server.py"], + type: "stdio" + }' \ + "$mcp_config" > "$tmp" + + mv "$tmp" "$mcp_config" + info "VS Code MCP server configuration updated for UnityMCP." +} + +verify_python_version() { + require_command python "python" + local version + version="$(python - <<'PY' +import sys +print("%d.%d.%d" % sys.version_info[:3]) +PY +)" + local major minor + IFS='.' read -r major minor _ <<< "$version" + if (( major < 3 || (major == 3 && minor < 12) )); then + error "Python 3.12+ is required. Detected version $version. Upgrade python before continuing." + exit 1 + fi + info "Python version $version satisfies requirement (>= 3.12)." +} + +print_next_steps() { + cat <<'EOT' + +Next steps: + 1. Launch Unity Hub and install a Unity Editor version 2021.3 LTS or newer. + 2. Open your Unity project and add the MCP for Unity Bridge package via: + Window > Package Manager > + > Add package from git URL... + https://github.com/CoplayDev/unity-mcp.git?path=/UnityMcpBridge + 3. In Unity, open Window > MCP for Unity and run Auto-Setup. Confirm the status shows Connected ✓. + 4. Open Visual Studio Code. The MCP server entry "unityMCP" is now configured. Reload if prompted. + 5. In VS Code, open the MCP client (e.g., Copilot / Claude Code) and issue a request such as "Create a tic-tac-toe game in 3D". The Unity MCP server should respond by operating inside your Unity project. + +Optional (Roslyn strict validation): + - Install NuGetForUnity and add Microsoft.CodeAnalysis + SQLitePCLRaw packages, then define USE_ROSLYN, OR + - Manually place Roslyn DLLs into Assets/Plugins and add USE_ROSLYN to Scripting Define Symbols. + +Troubleshooting tips: + - If VS Code cannot launch the server, ensure `uv` is on PATH and that ~/.local/bin is exported in your shell. + - To run the server manually: `uv --directory ~/.local/share/UnityMCP/UnityMcpServer/src run server.py` + - Verify the directory path in ~/.config/Code/User/mcp.json matches your installation. + +EOT +} + +main() { + if [[ ! -f /etc/arch-release ]]; then + error "This script is intended for Arch Linux." + exit 1 + fi + + info "Ensuring base dependencies are installed." + require_command sudo "sudo" + require_command pacman "pacman" + ensure_pacman_packages + verify_python_version + install_uv + ensure_unity_hub + sync_unity_mcp_repo + configure_vscode_mcp + print_next_steps + info "Setup complete. Follow the next steps above to finish configuration inside Unity." +} + +main "$@" diff --git a/mcp_readme.md b/mcp_readme.md new file mode 100644 index 0000000..6312f04 --- /dev/null +++ b/mcp_readme.md @@ -0,0 +1,189 @@ +## How It Works + +MCP for Unity connects your tools using two components: + +1. **MCP for Unity Bridge:** A Unity package running inside the Editor. (Installed via Package Manager). +2. **MCP for Unity Server:** A Python server that runs locally, communicating between the Unity Bridge and your MCP Client. (Installed automatically by the package on first run or via Auto-Setup; manual setup is available as a fallback). + +image + +--- + +## Installation ⚙️ + +### Prerequisites + + * **Python:** Version 3.12 or newer. [Download Python](https://www.python.org/downloads/) + * **Unity Hub & Editor:** Version 2021.3 LTS or newer. [Download Unity](https://unity.com/download) + * **uv (Python toolchain manager):** + ```bash + # macOS / Linux + curl -LsSf https://astral.sh/uv/install.sh | sh + + # Windows (PowerShell) + winget install --id=astral-sh.uv -e + + # Docs: https://docs.astral.sh/uv/getting-started/installation/ + ``` + + * **An MCP Client:** : [Claude Desktop](https://claude.ai/download) | [Claude Code](https://github.com/anthropics/claude-code) | [Cursor](https://www.cursor.com/en/downloads) | [Visual Studio Code Copilot](https://code.visualstudio.com/docs/copilot/overview) | [Windsurf](https://windsurf.com) | Others work with manual config + + *
[Optional] Roslyn for Advanced Script Validation + + For **Strict** validation level that catches undefined namespaces, types, and methods: + + **Method 1: NuGet for Unity (Recommended)** + 1. Install [NuGetForUnity](https://github.com/GlitchEnzo/NuGetForUnity) + 2. Go to `Window > NuGet Package Manager` + 3. Search for `Microsoft.CodeAnalysis`, select version 4.14.0, and install the package + 4. Also install package `SQLitePCLRaw.core` and `SQLitePCLRaw.bundle_e_sqlite3`. + 5. Go to `Player Settings > Scripting Define Symbols` + 6. Add `USE_ROSLYN` + 7. Restart Unity + + **Method 2: Manual DLL Installation** + 1. Download Microsoft.CodeAnalysis.CSharp.dll and dependencies from [NuGet](https://www.nuget.org/packages/Microsoft.CodeAnalysis.CSharp/) + 2. Place DLLs in `Assets/Plugins/` folder + 3. Ensure .NET compatibility settings are correct + 4. Add `USE_ROSLYN` to Scripting Define Symbols + 5. Restart Unity + + **Note:** Without Roslyn, script validation falls back to basic structural checks. Roslyn enables full C# compiler diagnostics with precise error reporting.
+ +--- +### 🚀 Arch Linux Quick Setup Script + +If you're on Arch Linux and use Visual Studio Code as your MCP client, run the helper script in `Bash/install_unity_mcp.sh` to install the MCP server dependencies, clone the latest `unity-mcp` repository, and configure `~/.config/Code/User/mcp.json` automatically: + +```bash +chmod +x Bash/install_unity_mcp.sh +./Bash/install_unity_mcp.sh +``` + +The script requires `sudo` access for `pacman` and optionally uses `yay` or `flatpak` to install Unity Hub. After it finishes, continue with the Unity-side steps below to import the MCP for Unity Bridge package inside your project. + +--- +### 🌟 Step 1: Install the Unity Package + +#### To install via Git URL + +1. Open your Unity project. +2. Go to `Window > Package Manager`. +3. Click `+` -> `Add package from git URL...`. +4. Enter: + ``` + https://github.com/CoplayDev/unity-mcp.git?path=/UnityMcpBridge + ``` +5. Click `Add`. +6. The MCP server is installed automatically by the package on first run or via Auto-Setup. If that fails, use Manual Configuration (below). + +#### To install via OpenUPM + +1. Install the [OpenUPM CLI](https://openupm.com/docs/getting-started-cli.html) +2. Open a terminal (PowerShell, Terminal, etc.) and navigate to your Unity project directory +3. Run `openupm add com.coplaydev.unity-mcp` + +**Note:** If you installed the MCP Server before Coplay's maintenance, you will need to uninstall the old package before re-installing the new one. + +### 🛠️ Step 2: Configure Your MCP Client +Connect your MCP Client (Claude, Cursor, etc.) to the Python server set up in Step 1 (auto) or via Manual Configuration (below). + +MCPForUnity-Readme-Image + +**Option A: Auto-Setup (Recommended for Claude/Cursor/VSC Copilot)** + +1. In Unity, go to `Window > MCP for Unity`. +2. Click `Auto-Setup`. +3. Look for a green status indicator 🟢 and "Connected ✓". *(This attempts to modify the MCP Client's config file automatically).* + +
Client-specific troubleshooting + + - **VSCode**: uses `Code/User/mcp.json` with top-level `servers.unityMCP` and `"type": "stdio"`. On Windows, MCP for Unity writes an absolute `uv.exe` (prefers WinGet Links shim) to avoid PATH issues. + - **Cursor / Windsurf** [(**help link**)](https://github.com/CoplayDev/unity-mcp/wiki/1.-Fix-Unity-MCP-and-Cursor,-VSCode-&-Windsurf): if `uv` is missing, the MCP for Unity window shows "uv Not Found" with a quick [HELP] link and a "Choose `uv` Install Location" button. + - **Claude Code** [(**help link**)](https://github.com/CoplayDev/unity-mcp/wiki/2.-Fix-Unity-MCP-and-Claude-Code): if `claude` isn't found, the window shows "Claude Not Found" with [HELP] and a "Choose Claude Location" button. Unregister now updates the UI immediately.
+ + +**Option B: Manual Configuration** + +If Auto-Setup fails or you use a different client: + +1. **Find your MCP Client's configuration file.** (Check client documentation). + * *Claude Example (macOS):* `~/Library/Application Support/Claude/claude_desktop_config.json` + * *Claude Example (Windows):* `%APPDATA%\Claude\claude_desktop_config.json` +2. **Edit the file** to add/update the `mcpServers` section, using the *exact* paths from Step 1. + +
+Click for Client-Specific JSON Configuration Snippets... + +**VSCode (all OS)** + +```json +{ + "servers": { + "unityMCP": { + "command": "uv", + "args": ["--directory","/UnityMcpServer/src","run","server.py"], + "type": "stdio" + } + } +} +``` + +**Linux:** + +```json +{ + "mcpServers": { + "UnityMCP": { + "command": "uv", + "args": [ + "run", + "--directory", + "/home/YOUR_USERNAME/.local/share/UnityMCP/UnityMcpServer/src", + "server.py" + ] + } + // ... other servers might be here ... + } +} +``` + +(Replace YOUR_USERNAME) + + +
+ +--- + +## Usage ▶️ + +1. **Open your Unity Project.** The MCP for Unity package should connect automatically. Check status via Window > MCP for Unity. + +2. **Start your MCP Client** (Claude, Cursor, etc.). It should automatically launch the MCP for Unity Server (Python) using the configuration from Installation Step 2. + +3. **Interact!** Unity tools should now be available in your MCP Client. + + Example Prompt: `Create a 3D player controller`, `Create a tic-tac-toe game in 3D`, `Create a cool shader and apply to a cube`. + +## Troubleshooting ❓ + +
+Click to view common issues and fixes... + +- **Unity Bridge Not Running/Connecting:** + - Ensure Unity Editor is open. + - Check the status window: Window > MCP for Unity. + - Restart Unity. +- **MCP Client Not Connecting / Server Not Starting:** + - **Verify Server Path:** Double-check the --directory path in your MCP Client's JSON config. It must exactly match the installation location: + - **Windows:** `%USERPROFILE%\AppData\Local\UnityMCP\UnityMcpServer\src` + - **macOS:** `~/Library/AppSupport/UnityMCP/UnityMcpServer\src` + - **Linux:** `~/.local/share/UnityMCP/UnityMcpServer\src` + - **Verify uv:** Make sure `uv` is installed and working (`uv --version`). + - **Run Manually:** Try running the server directly from the terminal to see errors: + ```bash + cd /path/to/your/UnityMCP/UnityMcpServer/src + uv run server.py + ``` +- **Auto-Configure Failed:** + - Use the Manual Configuration steps. Auto-configure might lack permissions to write to the MCP client's config file. \ No newline at end of file From 60517b4584473ca80860a3a6402605b4d1b04f7e Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 12 Oct 2025 14:46:55 +0200 Subject: [PATCH 12/15] feat: offline local transcribtion --- .gitignore | 3 +- .vscode/tasks.json | 21 ++ test_fw.srt | 4 + tools/transcribe_fw.py | 338 ++++++++++++++++++++++++++++++++ transcribe.sh | 430 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 795 insertions(+), 1 deletion(-) create mode 100644 .vscode/tasks.json create mode 100644 test_fw.srt create mode 100644 tools/transcribe_fw.py create mode 100644 transcribe.sh diff --git a/.gitignore b/.gitignore index 4750368..50fbb0c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ *.ogg* *.wav* *.m4a* -main_folder \ No newline at end of file +main_folder +models diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..3c4dc0e --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,21 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Transcribe tiny online smoke test", + "type": "shell", + "command": "bash", + "args": [ + "/home/kuhy/testsAndMisc/Bash/transcribe.sh", + "--online", + "-m", + "tiny" + ], + "isBackground": false, + "problemMatcher": [ + "$gcc" + ], + "group": "build" + } + ] +} \ No newline at end of file diff --git a/test_fw.srt b/test_fw.srt new file mode 100644 index 0000000..b9d1a04 --- /dev/null +++ b/test_fw.srt @@ -0,0 +1,4 @@ +1 +00:00:00,000 --> 00:00:02,760 +This is a quick test on faster with but run creep shun. + diff --git a/tools/transcribe_fw.py b/tools/transcribe_fw.py new file mode 100644 index 0000000..eb4ea0b --- /dev/null +++ b/tools/transcribe_fw.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +import argparse +import os +import shutil +import subprocess +import sys +import time +from datetime import timedelta +from typing import List, Optional + + +def format_timestamp(seconds: float) -> str: + td = timedelta(seconds=seconds) + # Ensure SRT format HH:MM:SS,mmm + total_seconds = int(td.total_seconds()) + hours = total_seconds // 3600 + minutes = (total_seconds % 3600) // 60 + secs = total_seconds % 60 + millis = int((seconds - int(seconds)) * 1000) + return f"{hours:02d}:{minutes:02d}:{secs:02d},{millis:03d}" + + +def write_srt(segments, srt_path: str): + with open(srt_path, "w", encoding="utf-8") as f: + for i, seg in enumerate(segments, start=1): + start = format_timestamp(seg.start) + end = format_timestamp(seg.end) + text = (seg.text or "").strip() + if not text: + continue + f.write(f"{i}\n{start} --> {end}\n{text}\n\n") + + +def write_txt(segments, txt_path: str): + with open(txt_path, "w", encoding="utf-8") as f: + for seg in segments: + text = (seg.text or "").strip() + if text: + f.write(text + "\n") + + +def write_srt_with_speakers(segments, labels: List[int], path: str): + with open(path, "w", encoding="utf-8") as f: + for i, (seg, lab) in enumerate(zip(segments, labels), start=1): + text = (seg.text or "").strip() + if not text: + continue + spk = f"SPK{lab+1}" + f.write(f"{i}\n{format_timestamp(seg.start)} --> {format_timestamp(seg.end)}\n[{spk}] {text}\n\n") + + +def write_txt_with_speakers(segments, labels: List[int], path: str): + with open(path, "w", encoding="utf-8") as f: + for seg, lab in zip(segments, labels): + text = (seg.text or "").strip() + if text: + spk = f"SPK{lab+1}" + f.write(f"[{spk}] {text}\n") + + +def write_rttm(segments, labels: List[int], path: str, file_id: str = "audio"): + # RTTM format: SPEAKER 1 + with open(path, "w", encoding="utf-8") as f: + for seg, lab in zip(segments, labels): + start = float(getattr(seg, "start", 0.0) or 0.0) + end = float(getattr(seg, "end", start) or start) + dur = max(0.0, end - start) + name = f"SPK{lab+1}" + f.write(f"SPEAKER {file_id} 1 {start:.3f} {dur:.3f} {name} \n") + + +def hhmmss(seconds: float) -> str: + seconds = max(0.0, float(seconds)) + total_seconds = int(seconds) + h = total_seconds // 3600 + m = (total_seconds % 3600) // 60 + s = total_seconds % 60 + return f"{h:02d}:{m:02d}:{s:02d}" + + +def get_media_duration(path: str) -> float | None: + """Try to get media duration in seconds using ffmpeg-python or ffprobe. + Returns None if unavailable. + """ + # Try ffmpeg-python first (if installed) which uses ffprobe under the hood + try: + import ffmpeg # type: ignore + + probe = ffmpeg.probe(path) + fmt = probe.get("format", {}) + if "duration" in fmt: + return float(fmt["duration"]) # type: ignore + except Exception: + pass + + # Fallback: call ffprobe directly if available + if shutil.which("ffprobe"): + try: + out = subprocess.check_output( + [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + path, + ], + stderr=subprocess.DEVNULL, + ) + return float(out.decode().strip()) + except Exception: + return None + return None + + +def _resample_linear(x, src_sr: int, tgt_sr: int): + import numpy as np + if src_sr == tgt_sr: + return x + ratio = float(tgt_sr) / float(src_sr) + n_out = max(1, int(round(x.shape[-1] * ratio))) + xp = np.linspace(0.0, 1.0, num=x.shape[-1], endpoint=False) + xq = np.linspace(0.0, 1.0, num=n_out, endpoint=False) + y = np.interp(xq, xp, x.astype(np.float32)) + return y.astype(np.float32) + + +def _kmeans_cosine(embs, k: int, iters: int = 50, seed: int = 0): + import numpy as np + rng = np.random.default_rng(seed) + X = np.asarray(embs, dtype=np.float32) + if X.ndim != 2 or X.shape[0] == 0: + return np.zeros((0,), dtype=np.int64) + # Normalize + X = X / (np.linalg.norm(X, axis=1, keepdims=True) + 1e-8) + # Init centroids as random samples + idxs = rng.choice(X.shape[0], size=min(k, X.shape[0]), replace=False) + C = X[idxs] + # If fewer samples than k, pad with random + if C.shape[0] < k: + pad = rng.standard_normal(size=(k - C.shape[0], X.shape[1])).astype(np.float32) + pad /= (np.linalg.norm(pad, axis=1, keepdims=True) + 1e-8) + C = np.concatenate([C, pad], axis=0) + for _ in range(iters): + # Assign by cosine similarity (maximize dot product) + sims = X @ C.T # (n, k) + labels = sims.argmax(axis=1) + newC = np.zeros_like(C) + for j in range(k): + sel = X[labels == j] + if sel.shape[0] == 0: + newC[j] = C[j] + else: + v = sel.mean(axis=0) + v /= (np.linalg.norm(v) + 1e-8) + newC[j] = v + if np.allclose(newC, C, atol=1e-4): + break + C = newC + return labels + + +def diarize_segments(audio_path: str, segments, num_speakers: int = 2) -> Optional[list]: + """Simple diarization: compute speaker embeddings per segment and cluster with KMeans. + Returns a list of speaker labels aligned with segments, or None on failure. + """ + try: + import numpy as np + import soundfile as sf + from speechbrain.pretrained import EncoderClassifier + import torch + except Exception as e: + print(f"[WARN] Diarization dependencies missing ({e}); skipping speaker labels.", file=sys.stderr) + return None + + # Load audio + try: + wav, sr = sf.read(audio_path, dtype="float32", always_2d=False) + except Exception as e: + print(f"[WARN] Could not read audio for diarization: {e}", file=sys.stderr) + return None + if wav.ndim == 2: # mixdown + wav = wav.mean(axis=1) + # Resample to 16k for ECAPA + wav16 = _resample_linear(wav, sr, 16000) + + # Load speaker embedding model (CPU is fine) + try: + classifier = EncoderClassifier.from_hparams( + source="speechbrain/spkrec-ecapa-voxceleb", + run_opts={"device": "cpu"}, + savedir=os.path.join(os.path.expanduser("~"), ".cache", "speechbrain_ecapa") + ) + except Exception as e: + print(f"[WARN] Could not load speaker embedding model: {e}", file=sys.stderr) + return None + + embs = [] + # Extract embedding per segment window + for seg in segments: + s = float(getattr(seg, "start", 0.0) or 0.0) + e = float(getattr(seg, "end", s) or s) + if e <= s: + e = s + 0.2 # minimal window + # Convert to samples in 16k + i0 = int(s * 16000) + i1 = int(e * 16000) + # Add small margins to help very short segments + pad = int(0.05 * 16000) + i0 = max(0, i0 - pad) + i1 = min(len(wav16), i1 + pad) + if i1 - i0 < 1600: # <0.1s, too short; expand if possible + i1 = min(len(wav16), i0 + 1600) + segment_wav = torch.tensor(wav16[i0:i1]).unsqueeze(0) + with torch.no_grad(): + emb = classifier.encode_batch(segment_wav).squeeze(0).squeeze(0).cpu().numpy() + embs.append(emb.astype("float32")) + + if len(embs) == 0: + return None + # Cluster + labels = _kmeans_cosine(embs, k=max(1, int(num_speakers))) + return labels.tolist() + + +def main(): + parser = argparse.ArgumentParser(description="Transcribe audio with faster-whisper and write .txt and .srt") + parser.add_argument("input", help="Path to audio/video file") + parser.add_argument("--model", default=os.environ.get("FW_MODEL", "large-v3"), help="Model size or path (default: large-v3)") + parser.add_argument("--language", default=None, help="Language code (e.g., en). Leave None for auto-detect") + parser.add_argument("--device", default=os.environ.get("FW_DEVICE", "auto"), choices=["auto", "cpu", "cuda"], help="Device to run on") + parser.add_argument("--compute-type", dest="compute_type", default=os.environ.get("FW_COMPUTE", "auto"), help="Compute type (auto,int8,float16,float32,int8_float16,etc.)") + parser.add_argument("--outdir", default=None, help="Output directory (default: next to input)") + parser.add_argument("--no-progress", action="store_true", help="Disable live progress output") + parser.add_argument("--diarize", action="store_true", help="Enable speaker diarization (labels)") + parser.add_argument("--num-speakers", type=int, default=int(os.environ.get("FW_NUM_SPEAKERS", "2")), help="Assumed number of speakers (default: 2)") + args = parser.parse_args() + + try: + from faster_whisper import WhisperModel + except Exception as e: + print("[ERROR] faster-whisper is not installed in this environment.", file=sys.stderr) + print(str(e), file=sys.stderr) + return 2 + + inp = os.path.abspath(args.input) + if not os.path.exists(inp): + print(f"[ERROR] Input file not found: {inp}", file=sys.stderr) + return 2 + + outdir = os.path.abspath(args.outdir or os.path.dirname(inp) or ".") + os.makedirs(outdir, exist_ok=True) + base = os.path.splitext(os.path.basename(inp))[0] + srt_path = os.path.join(outdir, base + ".srt") + txt_path = os.path.join(outdir, base + ".txt") + + # Device and compute_type heuristics + device = args.device + compute_type = args.compute_type + if device == "auto": + device = "cpu" + if compute_type == "auto": + # Prefer accuracy over speed by default + compute_type = "float16" if device == "cuda" else "float32" + + print(f"[INFO] Loading model='{args.model}', device='{device}', compute_type='{compute_type}'") + model = WhisperModel(args.model, device=device, compute_type=compute_type) + + # Transcription with live progress + total_duration = get_media_duration(inp) + if total_duration: + print(f"[INFO] Media duration: {hhmmss(total_duration)}") + start_ts = time.time() + + iter_segments, info = model.transcribe(inp, language=args.language) + collected = [] + processed = 0.0 + last_print = 0.0 + tty = sys.stderr.isatty() + for seg in iter_segments: + collected.append(seg) + # Update processed time from segment end if available + if getattr(seg, "end", None) is not None: + processed = max(processed, float(seg.end)) + now = time.time() + # Print each segment or throttle to ~5 per second + if not args.no_progress and (tty or (now - last_print) >= 0.2): + last_print = now + if total_duration and total_duration > 0: + pct = max(0.0, min(100.0, (processed / total_duration) * 100.0)) + elapsed = now - start_ts + eta = None + if processed > 0: + rate = processed / max(1e-6, elapsed) + remaining = max(0.0, total_duration - processed) + eta = remaining / max(1e-6, rate) + line = f"[PROGRESS] {hhmmss(processed)} / {hhmmss(total_duration)} ({pct:5.1f}%)" + if eta is not None and eta < 60 * 60 * 24: # cap unrealistic values + line += f" ETA ~{hhmmss(eta)}" + else: + line = f"[PROGRESS] processed {hhmmss(processed)}" + if tty: + print("\r" + line, end="", file=sys.stderr, flush=True) + else: + print(line, file=sys.stderr, flush=True) + + # Finish progress line + if not args.no_progress and sys.stderr.isatty(): + print("", file=sys.stderr) # newline + + print(f"[INFO] Detected language: {getattr(info, 'language', None)} (prob={getattr(info, 'language_probability', None)})") + print(f"[INFO] Segments: {len(collected)}") + + # Optionally diarize + if args.diarize: + labels = diarize_segments(inp, collected, num_speakers=args.num_speakers) + if labels is not None and len(labels) == len(collected): + diar_srt = os.path.join(outdir, base + ".diar.srt") + diar_txt = os.path.join(outdir, base + ".diar.txt") + rttm_path = os.path.join(outdir, base + ".rttm") + write_srt_with_speakers(collected, labels, diar_srt) + write_txt_with_speakers(collected, labels, diar_txt) + write_rttm(collected, labels, rttm_path, file_id=base) + print(f"[OK] Wrote: {diar_txt}\n[OK] Wrote: {diar_srt}\n[OK] Wrote: {rttm_path}") + else: + print("[WARN] Diarization failed or returned mismatched labels; writing plain outputs.", file=sys.stderr) + + # Write base outputs + write_txt(collected, txt_path) + write_srt(collected, srt_path) + print(f"[OK] Wrote: {txt_path}\n[OK] Wrote: {srt_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/transcribe.sh b/transcribe.sh new file mode 100644 index 0000000..e1cce85 --- /dev/null +++ b/transcribe.sh @@ -0,0 +1,430 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Transcribe an audio file using faster-whisper with automatic setup. +# - Creates Python venv in .venv +# - Installs ffmpeg and espeak-ng (best-effort) for test audio generation +# - Installs faster-whisper (and CUDA stack if NVIDIA is present) +# - Runs tools/transcribe_fw.py to produce .txt and .srt next to the input + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$SCRIPT_DIR" +TOOLS_DIR="$PROJECT_DIR/tools" +PY_RUNNER="$TOOLS_DIR/transcribe_fw.py" +VENV_DIR="$PROJECT_DIR/.venv" + +usage() { + cat </dev/null 2>&1; then echo apt; return; fi + if command -v dnf >/dev/null 2>&1; then echo dnf; return; fi + if command -v yum >/dev/null 2>&1; then echo yum; return; fi + if command -v pacman >/dev/null 2>&1; then echo pacman; return; fi + if command -v zypper >/dev/null 2>&1; then echo zypper; return; fi + echo none +} + +has_libcublas12() { + # Common system locations + for d in \ + /usr/lib \ + /usr/lib64 \ + /usr/local/cuda/lib64 \ + /usr/local/cuda-12*/lib64 \ + /opt/cuda/lib64 \ + /opt/cuda/targets/x86_64-linux/lib; do + [[ -e "$d/libcublas.so.12" ]] && return 0 || true + done + # venv-provided NVIDIA CUDA libs + if [[ -x "$VENV_DIR/bin/python" ]]; then + local pyver + pyver="$($VENV_DIR/bin/python -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || true)" + if [[ -n "$pyver" ]]; then + for d in "$VENV_DIR/lib/python$pyver/site-packages/nvidia/cublas/lib" \ + "$VENV_DIR/lib/python$pyver/site-packages/nvidia/cudnn/lib" \ + "$VENV_DIR/lib/python$pyver/site-packages/nvidia/cuda_runtime/lib"; do + [[ -e "$d/libcublas.so.12" ]] && return 0 || true + done + fi + fi + return 1 +} + +ensure_cuda_runtime() { + local mgr; mgr="$(detect_pkg_mgr)" + if [[ $OFFLINE -eq 1 ]]; then + if has_libcublas12; then return 0; fi + echo "CUDA runtime (libcublas.so.12) not found and offline mode is enabled. Install CUDA 12 runtime or rerun with --online." >&2 + exit 6 + fi + if has_libcublas12; then + return 0 + fi + if ! command -v sudo >/dev/null 2>&1; then + log "sudo not found; skipping CUDA runtime install attempt." + else + log "CUDA cuBLAS 12 not found; attempting to install CUDA runtime (manager: $mgr)" + set +e + case "$mgr" in + pacman) + sudo pacman -Sy --noconfirm cuda cudnn || true ;; + apt) + sudo apt-get update -y || true + sudo apt-get install -y nvidia-cuda-toolkit || true ;; + dnf|yum) + sudo "$mgr" install -y cuda cudnn || true ;; + zypper) + sudo zypper install -y cuda cudnn || true ;; + *) log "Unknown package manager; cannot install CUDA automatically." ;; + esac + set -e + fi + # Re-check + if ! has_libcublas12; then + echo "CUDA runtime (libcublas.so.12) not found after attempted install. Please install CUDA 12 toolkit/runtime and re-run." >&2 + exit 6 + fi +} + +install_system_deps() { + have_cmd() { command -v "$1" >/dev/null 2>&1; } + local need_ffmpeg=0 need_espeak=0 + have_cmd ffmpeg || need_ffmpeg=1 + have_cmd espeak-ng || need_espeak=1 + + # If diarization requested and online, we may also try to ensure libsndfile + local need_libsndfile=0 + if [[ "${FW_DIARIZE:-}" == "1" ]]; then + # Heuristic: check common library file + if [[ ! -e /usr/lib/x86_64-linux-gnu/libsndfile.so && ! -e /usr/lib/libsndfile.so && ! -e /usr/lib64/libsndfile.so ]]; then + need_libsndfile=1 + fi + fi + + if [[ $need_ffmpeg -eq 0 && $need_espeak -eq 0 && $need_libsndfile -eq 0 ]]; then + log "System deps present: ffmpeg, espeak-ng${FW_DIARIZE:+, libsndfile}" + return 0 + fi + + if [[ $OFFLINE -eq 1 ]]; then + echo "Missing system dependencies (ffmpeg/espeak-ng) but running in offline mode. Install them or rerun with --online." >&2 + exit 5 + fi + + local mgr; mgr="$(detect_pkg_mgr)" + log "Detected package manager: $mgr (installing missing: $([[ $need_ffmpeg -eq 1 ]] && echo ffmpeg )$([[ $need_espeak -eq 1 ]] && echo espeak-ng )$([[ $need_libsndfile -eq 1 ]] && echo libsndfile))" + + if ! command -v sudo >/dev/null 2>&1; then + log "sudo not found; skipping system package installation attempt." + return 0 + fi + + # Avoid exiting on install errors; continue best-effort + set +e + case "$mgr" in + apt) + sudo apt-get update -y || log "apt-get update failed; continuing" + pkgs=(python3-venv python3-pip) + [[ $need_ffmpeg -eq 1 ]] && pkgs+=(ffmpeg) + [[ $need_espeak -eq 1 ]] && pkgs+=(espeak-ng) + if [[ $need_libsndfile -eq 1 ]]; then + # Try both names across releases + pkgs+=(libsndfile1) + sudo apt-get install -y libsndfile1 || true + # If that failed, try libsndfile2 (newer distros) + sudo apt-get install -y libsndfile2 || true + fi + sudo apt-get install -y "${pkgs[@]}" || log "apt-get install failed; continuing" ;; + dnf) + pkgs=(python3-venv python3-pip) + [[ $need_ffmpeg -eq 1 ]] && pkgs+=(ffmpeg) + [[ $need_espeak -eq 1 ]] && pkgs+=(espeak-ng) + [[ $need_libsndfile -eq 1 ]] && pkgs+=(libsndfile) + sudo dnf install -y "${pkgs[@]}" || log "dnf install failed; continuing" ;; + yum) + pkgs=(python3-venv python3-pip) + [[ $need_ffmpeg -eq 1 ]] && pkgs+=(ffmpeg) + [[ $need_espeak -eq 1 ]] && pkgs+=(espeak-ng) + [[ $need_libsndfile -eq 1 ]] && pkgs+=(libsndfile) + sudo yum install -y "${pkgs[@]}" || log "yum install failed; continuing" ;; + pacman) + pkgs=(python-virtualenv python-pip) + [[ $need_ffmpeg -eq 1 ]] && pkgs+=(ffmpeg) + [[ $need_espeak -eq 1 ]] && pkgs+=(espeak-ng) + [[ $need_libsndfile -eq 1 ]] && pkgs+=(libsndfile) + sudo pacman -Sy --noconfirm "${pkgs[@]}" || log "pacman install failed; continuing" ;; + zypper) + pkgs=(python311-virtualenv python311-pip) + [[ $need_ffmpeg -eq 1 ]] && pkgs+=(ffmpeg) + [[ $need_espeak -eq 1 ]] && pkgs+=(espeak-ng) + [[ $need_libsndfile -eq 1 ]] && pkgs+=(libsndfile1) + sudo zypper install -y "${pkgs[@]}" || log "zypper install failed; continuing" ;; + *) + log "Unknown package manager; please ensure ffmpeg and espeak-ng are installed." ;; + esac + set -e +} + +setup_venv() { + if [[ ! -d "$VENV_DIR" ]]; then + log "Creating venv at $VENV_DIR" + python3 -m venv "$VENV_DIR" + fi + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" + if [[ $OFFLINE -eq 0 ]]; then + python -m pip install --upgrade pip wheel setuptools + fi +} + +install_python_deps() { + # Install deps; if NVIDIA GPU is present, prefer CUDA-capable stack (cu12) + local has_nvidia_flag="${1:-0}" + log "Installing faster-whisper and dependencies" + export PIP_DISABLE_PIP_VERSION_CHECK=1 + export PIP_DEFAULT_TIMEOUT=${PIP_DEFAULT_TIMEOUT:-20} + if [[ $OFFLINE -eq 1 ]]; then + # Offline: do not install, just verify modules + if ! python -c 'import faster_whisper' >/dev/null 2>&1; then + echo "Python dependency 'faster_whisper' not found in offline mode. Run with --online to install." >&2 + exit 7 + fi + # If diarization requested offline, check for its deps too (warn-only) + if [[ "${FW_DIARIZE:-}" == "1" ]]; then + python - <<'PY' || true +try: + import soundfile, speechbrain, torch # noqa: F401 +except Exception as e: + print(f"[WARN] Diarization deps missing offline ({e}); speaker labels will be skipped.") +PY + fi + return 0 + fi + if [[ "$has_nvidia_flag" -eq 1 ]]; then + # If ctranslate2 is not installed, attempt CUDA-enabled wheel (quiet, with fallback) + if ! "$VENV_DIR/bin/python" -c 'import ctranslate2' >/dev/null 2>&1; then + log "Installing CUDA-enabled CTranslate2 (cu12 wheel)" + python -m pip install -q --retries 1 --upgrade "ctranslate2<5,>=4.0" --extra-index-url https://download.opennmt.net/ctranslate2/cu12 || \ + log "Warning: could not reach cu12 wheel index; will proceed with available ctranslate2" + fi + # Ensure NVIDIA CUDA 12 runtime libs are available inside the venv + python -m pip install -q --retries 1 --upgrade nvidia-cublas-cu12 nvidia-cuda-runtime-cu12 nvidia-cudnn-cu12 || \ + log "Warning: failed to install NVIDIA cu12 runtime libs via pip" + fi + python -m pip install -q --retries 1 --upgrade faster-whisper ffmpeg-python + + # If diarization requested and online, install its Python deps best-effort + if [[ "${FW_DIARIZE:-}" == "1" ]]; then + python -m pip install -q --retries 1 --upgrade soundfile speechbrain || \ + log "Warning: failed to install soundfile/speechbrain" + # Torch and torchaudio CPU wheels (force to avoid mismatched CUDA builds) + python -m pip install -q --retries 1 --upgrade --force-reinstall --index-url https://download.pytorch.org/whl/cpu torch torchaudio || \ + log "Warning: failed to install torch/torchaudio CPU wheels" + fi + python - <<'PY' +import sys +print(f"[PY] Python {sys.version.split()[0]} dependencies installed.") +PY +} + +ensure_runner() { + if [[ ! -f "$PY_RUNNER" ]]; then + echo "Runner not found: $PY_RUNNER" >&2 + exit 3 + fi +} + +generate_test_audio() { + local tmpwav + tmpwav="${PROJECT_DIR}/test_fw.wav" + if command -v espeak-ng >/dev/null 2>&1; then + log "Generating test audio via espeak-ng -> $tmpwav" >&2 + espeak-ng -w "$tmpwav" "This is a quick test of faster whisper transcription." >/dev/null 2>&1 || true + fi + # If espeak-ng failed or not present, try espeak + if [[ ! -s "$tmpwav" ]] && command -v espeak >/dev/null 2>&1; then + log "espeak-ng unavailable or failed; trying espeak -> $tmpwav" >&2 + espeak -w "$tmpwav" "This is a quick test of faster whisper transcription." >/dev/null 2>&1 || true + fi + # Fallback: generate tone via Python stdlib (no external deps) + if [[ ! -s "$tmpwav" ]]; then + log "Generating 3s 1kHz WAV via Python stdlib -> $tmpwav" >&2 + python3 -c 'import sys,wave,math,array;outfile=sys.argv[1];fr=16000;dur=3;freq=1000.0;ampl=0.3;n=fr*dur;data=array.array("h",[int(max(-1.0,min(1.0,ampl*math.sin(2*math.pi*freq*(i/fr))))*32767) for i in range(n)]);wf=wave.open(outfile,"w");wf.setnchannels(1);wf.setsampwidth(2);wf.setframerate(fr);wf.writeframes(data.tobytes());wf.close()' "$tmpwav" || true + fi + # Final fallback: tone via ffmpeg + if [[ ! -s "$tmpwav" ]]; then + log "Creating a 3s sine tone WAV via ffmpeg -> $tmpwav" >&2 + ffmpeg -f lavfi -i sine=frequency=1000:duration=3 -ar 16000 -ac 1 -f wav -y "$tmpwav" >/dev/null 2>&1 || true + fi + echo "$tmpwav" +} + +prepare_model() { + # Download a model for offline use into MODEL_DIR + local name="$1" + mkdir -p "$MODEL_DIR" + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" + log "Preparing model '$name' into $MODEL_DIR" + python - <&2 + exit 2 + fi + install_python_deps 0 + export FW_PREPARE_NAME="$PREPARE_MODEL" + export FW_MODEL_DIR="$MODEL_DIR" + prepare_model "$PREPARE_MODEL" + log "Model '$PREPARE_MODEL' downloaded to $MODEL_DIR" + exit 0 + fi + + # Detect NVIDIA GPU and enforce CUDA if present + has_nvidia=0 + if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then + has_nvidia=1 + fi + install_python_deps "$has_nvidia" + ensure_runner + + local input="$INPUT_FILE" + if [[ -z "$input" ]]; then + input="$(generate_test_audio)" + if [[ ! -s "$input" ]]; then + echo "Failed to generate test audio. Please provide an audio file." >&2 + exit 4 + fi + fi + + if [[ ! -f "$input" ]]; then + echo "Input file not found: $input" >&2 + exit 2 + fi + + local args=("$input" "--model" "$MODEL") + [[ -n "$LANGUAGE" ]] && args+=("--language" "$LANGUAGE") + [[ -n "$OUTDIR" ]] && args+=("--outdir" "$OUTDIR") + + # Pass diarization via env if requested + if [[ "${FW_DIARIZE:-}" == "1" ]]; then + args+=("--diarize") + if [[ -n "${FW_NUM_SPEAKERS:-}" ]]; then + args+=("--num-speakers" "${FW_NUM_SPEAKERS}") + fi + fi + + if [[ $has_nvidia -eq 1 ]]; then + ensure_cuda_runtime + # Export common CUDA paths in case the env lacks them + export CUDA_HOME="${CUDA_HOME:-/usr/local/cuda}" + # Include system and possible venv-provided CUDA libs + local pyver venv_cuda_paths="" + if [[ -x "$VENV_DIR/bin/python" ]]; then + pyver="$($VENV_DIR/bin/python -c 'import sys;print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || true)" + if [[ -n "$pyver" ]]; then + venv_cuda_paths="$VENV_DIR/lib/python$pyver/site-packages/nvidia/cublas/lib:$VENV_DIR/lib/python$pyver/site-packages/nvidia/cudnn/lib:$VENV_DIR/lib/python$pyver/site-packages/nvidia/cuda_runtime/lib" + fi + fi + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:${CUDA_HOME}/lib64:/usr/lib/x86_64-linux-gnu:/opt/cuda/lib64:/opt/cuda/targets/x86_64-linux/lib:${venv_cuda_paths}" + export PATH="${PATH}:${CUDA_HOME}/bin" + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" + python -c 'from faster_whisper import WhisperModel; WhisperModel("tiny", device="cuda", compute_type="float16"); print("[PY] CUDA test init succeeded.")' || { echo "CUDA environment check failed. Aborting as requested." >&2; exit 6; } + args+=("--device" "cuda") + fi + + log "Transcribing: $input" + # shellcheck disable=SC1091 + source "$VENV_DIR/bin/activate" + if [[ $has_nvidia -eq 1 ]]; then + if ! python "$PY_RUNNER" "${args[@]}"; then + echo "CUDA execution requested due to detected NVIDIA GPU, but it failed. Aborting as requested (no CPU fallback)." >&2 + exit 6 + fi + else + # Offline: prefer local directory if present; otherwise use cache without network + if [[ $OFFLINE -eq 1 ]]; then + local local_model_path="" + if [[ -d "$MODEL" ]]; then + local_model_path="$MODEL" + elif [[ -d "$MODEL_DIR/$MODEL" ]]; then + local_model_path="$MODEL_DIR/$MODEL" + fi + if [[ -n "$local_model_path" ]]; then + args=("$input" "--model" "$local_model_path") + [[ -n "$LANGUAGE" ]] && args+=("--language" "$LANGUAGE") + [[ -n "$OUTDIR" ]] && args+=("--outdir" "$OUTDIR") + fi + fi + python "$PY_RUNNER" "${args[@]}" + fi +} + +main "$@" + From 06c8121c79ece96992d2e6c78d201bc469851e2b Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 12 Oct 2025 14:51:41 +0200 Subject: [PATCH 13/15] feat: automatically change audio to supporet diarization --- tools/transcribe_fw.py | 70 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/tools/transcribe_fw.py b/tools/transcribe_fw.py index eb4ea0b..48eff05 100644 --- a/tools/transcribe_fw.py +++ b/tools/transcribe_fw.py @@ -7,8 +7,6 @@ import sys import time from datetime import timedelta from typing import List, Optional - - def format_timestamp(seconds: float) -> str: td = timedelta(seconds=seconds) # Ensure SRT format HH:MM:SS,mmm @@ -162,6 +160,41 @@ def _kmeans_cosine(embs, k: int, iters: int = 50, seed: int = 0): return labels +def _ffmpeg_transcode_to_wav16_mono(src_path: str) -> Optional[str]: + """If ffmpeg is available, transcode input to a temporary 16k mono WAV and return its path.""" + if not shutil.which("ffmpeg"): + return None + import tempfile + tmp = tempfile.NamedTemporaryFile(prefix="fw_diar_", suffix=".wav", delete=False) + tmp_path = tmp.name + tmp.close() + # Run ffmpeg quietly + cmd = [ + "ffmpeg", + "-y", + "-v", + "error", + "-i", + src_path, + "-ac", + "1", + "-ar", + "16000", + "-f", + "wav", + tmp_path, + ] + try: + subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return tmp_path + except Exception: + try: + os.unlink(tmp_path) + except Exception: + pass + return None + + def diarize_segments(audio_path: str, segments, num_speakers: int = 2) -> Optional[list]: """Simple diarization: compute speaker embeddings per segment and cluster with KMeans. Returns a list of speaker labels aligned with segments, or None on failure. @@ -169,18 +202,33 @@ def diarize_segments(audio_path: str, segments, num_speakers: int = 2) -> Option try: import numpy as np import soundfile as sf - from speechbrain.pretrained import EncoderClassifier + # Use non-deprecated import path + from speechbrain.inference import EncoderClassifier import torch except Exception as e: print(f"[WARN] Diarization dependencies missing ({e}); skipping speaker labels.", file=sys.stderr) return None # Load audio + temp_to_cleanup: Optional[str] = None try: wav, sr = sf.read(audio_path, dtype="float32", always_2d=False) except Exception as e: - print(f"[WARN] Could not read audio for diarization: {e}", file=sys.stderr) - return None + # Try ffmpeg transcoding fallback + alt = _ffmpeg_transcode_to_wav16_mono(audio_path) + if alt is None: + print(f"[WARN] Could not read audio for diarization and no ffmpeg fallback available: {e}", file=sys.stderr) + return None + try: + wav, sr = sf.read(alt, dtype="float32", always_2d=False) + temp_to_cleanup = alt + except Exception as e2: + print(f"[WARN] Could not read transcoded audio for diarization: {e2}", file=sys.stderr) + try: + os.unlink(alt) + except Exception: + pass + return None if wav.ndim == 2: # mixdown wav = wav.mean(axis=1) # Resample to 16k for ECAPA @@ -191,10 +239,15 @@ def diarize_segments(audio_path: str, segments, num_speakers: int = 2) -> Option classifier = EncoderClassifier.from_hparams( source="speechbrain/spkrec-ecapa-voxceleb", run_opts={"device": "cpu"}, - savedir=os.path.join(os.path.expanduser("~"), ".cache", "speechbrain_ecapa") + savedir=os.path.join(os.path.expanduser("~"), ".cache", "speechbrain_ecapa"), ) except Exception as e: print(f"[WARN] Could not load speaker embedding model: {e}", file=sys.stderr) + if temp_to_cleanup: + try: + os.unlink(temp_to_cleanup) + except Exception: + pass return None embs = [] @@ -222,6 +275,11 @@ def diarize_segments(audio_path: str, segments, num_speakers: int = 2) -> Option return None # Cluster labels = _kmeans_cosine(embs, k=max(1, int(num_speakers))) + if temp_to_cleanup: + try: + os.unlink(temp_to_cleanup) + except Exception: + pass return labels.tolist() From 158a1c7b0187bc1f434eb642f25c2b93f2557fed Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 12 Oct 2025 18:57:55 +0200 Subject: [PATCH 14/15] fix: getting rnnoise model --- README_clean_audio.md | 104 +++++++++ clean_audio.sh | 390 ++++++++++++++++++++++++++++++++++ get_rnnoise_model.sh | 190 +++++++++++++++++ install_ffmpeg_with_arnndn.sh | 130 ++++++++++++ 4 files changed, 814 insertions(+) create mode 100644 README_clean_audio.md create mode 100755 clean_audio.sh create mode 100755 get_rnnoise_model.sh create mode 100755 install_ffmpeg_with_arnndn.sh diff --git a/README_clean_audio.md b/README_clean_audio.md new file mode 100644 index 0000000..c0bc61f --- /dev/null +++ b/README_clean_audio.md @@ -0,0 +1,104 @@ +# clean_audio.sh — automatic speech cleaning (FFmpeg) + +This script batch‑cleans noisy speech recordings with ffmpeg using simple, reliable filters tuned for ASR (e.g., faster‑whisper). By default it REQUIRES RNNoise (arnndn) and will try to auto‑discover or download a model. You can opt‑in to fallback filters with `--allow-fallback`. + +## Install + +- Required: ffmpeg. Most distros: `sudo pacman -S ffmpeg` or `sudo apt install ffmpeg`. +- Recommended: ffmpeg with `arnndn` filter and an RNNoise model file (e.g., from Mozilla RNNoise community models). The script will auto-detect common model locations or download one via `Bash/get_rnnoise_model.sh`. You can pass a specific model with `-m /path/to/model.nn`. + +Make executable: + +```bash +chmod +x Bash/clean_audio.sh +``` + +## Quick start + +- Single file, default ASR preset (16k mono, denoise, high‑pass, limiter): +```bash +Bash/clean_audio.sh path/to/file.wav +``` +This produces `path/to/file_clean.wav`. + +- Whole folder, 4 parallel jobs, output to `cleaned/`: +```bash +Bash/clean_audio.sh path/to/folder -O cleaned -j 4 +``` + +- Use an RNNoise model explicitly (if your ffmpeg has arnndn): +```bash +Bash/clean_audio.sh input.wav -m models/rnnoise_model.nn +``` +If you omit `-m`, the script will look in common locations; if not found, it will attempt a download via `Bash/get_rnnoise_model.sh`. + +Advanced options and compatibility: +- The cleaner requires RNNoise by default. To allow non-ML fallback filters (afftdn), add `--allow-fallback`. +- The script uses advanced filter settings when available (e.g., afftdn with `md`). If your ffmpeg build lacks these options, it will error with guidance. Add `--no-advanced` (or `--compat`) to avoid such params. + +- Podcast preset (adds dynamics and loudness leveling): +```bash +Bash/clean_audio.sh input.wav --preset podcast +``` + +## Options + +```text +Usage: clean_audio.sh [options] + +Options: + -O, --out-dir DIR Output directory (default: alongside input file). + -e, --ext EXT Output extension/container: wav|flac (default: wav). + -m, --model PATH RNNoise model file for arnndn; falls back to afftdn if unavailable. + --no-ml Do not use arnndn even if model is provided; use afftdn. + --preset NAME asr (default) | podcast | aggressive + -j, --jobs N Parallel jobs for directory mode (default: 1). + -f, --force Overwrite outputs if they exist. + -q, --quiet Reduce ffmpeg logging noise. + --lowpass FREQ Optional low-pass cutoff (e.g., 8000). Disabled by default. + --suffix SUF Suffix for output basename (default: _clean). +``` + +## Designed for ASR (faster‑whisper) + +Default output format is mono, 16 kHz, PCM 16‑bit WAV—ideal for most Whisper/faster‑whisper pipelines. You can feed the cleaned files directly into your transcription step. + +If you prefer FLAC to save space without quality loss: +```bash +Bash/clean_audio.sh input.wav -e flac -O cleaned +``` + +## Presets + +- asr (default): light, ASR‑friendly cleanup; prevents clipping. +- podcast: adds gentle dynamics and approximate loudness normalization (single‑pass `loudnorm`). +- aggressive: heavier gate/dynamics; can suppress background more, but may slightly hurt ASR accuracy—use sparingly. + +## Tips + +- If you see artifacts from RNNoise, try without a model (uses `afftdn`), or add a low‑pass (e.g., `--lowpass 8000`). +- For extremely boomy bar recordings, raise high‑pass by editing `HIGHPASS` in the script or add `--lowpass`. +- If your ffmpeg lacks `arnndn`, you can install a newer build or keep the fallback (afftdn works fine for many cases). + - If your ffmpeg is missing features, you can use the helper: +```bash +chmod +x Bash/install_ffmpeg_with_arnndn.sh +Bash/install_ffmpeg_with_arnndn.sh +``` +It will suggest distro options or build FFmpeg from source with `--enable-librnnoise`. + + RNNoise model downloader helper: + ```bash + chmod +x Bash/get_rnnoise_model.sh + Bash/get_rnnoise_model.sh --yes + ``` + This saves a model into `Bash/models/` which the cleaner will auto-discover. + +## Troubleshooting + +- “arnndn not available”: Your ffmpeg wasn’t built with it. The script will use `afftdn` instead. +- Output sounds thin: lower the high‑pass (edit `HIGHPASS=80` in script to `60`) or remove low‑pass. +- Level too low/high: choose the `podcast` preset for auto leveling, or add your own `loudnorm` in post. + +## License + +This helper script is provided under the repository’s LICENSE. diff --git a/clean_audio.sh b/clean_audio.sh new file mode 100755 index 0000000..023b038 --- /dev/null +++ b/clean_audio.sh @@ -0,0 +1,390 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# clean_audio.sh — Fully automatic audio cleaner for speech (ASR-friendly) +# +# - Default preset is tuned for ASR (faster-whisper): +# mono, 16 kHz, high-pass filter, denoise (RNNoise arnndn by default if model found/provided; else afftdn), +# peak limiting to -0.5 dBFS. No aggressive gating/compression by default. +# - Optional "podcast" preset adds gentle dynamics and loudness leveling. +# - Accepts single files or directories (recursively). +# - Optional parallel processing. +# +# Dependencies: ffmpeg (arnndn filter recommended for best results) +# Optional: an RNNoise model file for arnndn (auto-discovered if present; otherwise falls back to afftdn) +# +# Usage examples: +# Bash/clean_audio.sh input.wav # -> input_clean.wav (same folder) +# Bash/clean_audio.sh input.wav -O out_dir # -> out_dir/input_clean.wav +# Bash/clean_audio.sh input_dir -O cleaned/ -j 4 # -> processes all audio files in dir +# Bash/clean_audio.sh input.wav -m models/rn.nn # -> use RNNoise model +# Bash/clean_audio.sh input.wav --preset podcast # -> add dynamics leveler +# + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +print_usage() { + cat < [options] + +Options: + -O, --out-dir DIR Output directory (default: alongside input file). + -e, --ext EXT Output extension/container: wav|flac (default: wav). + -m, --model PATH RNNoise model file for arnndn; required by default unless --allow-fallback. + --no-ml Do not use arnndn even if model is provided (requires --allow-fallback). + --preset NAME asr (default) | podcast | aggressive + -j, --jobs N Parallel jobs for directory mode (default: 1). + -f, --force Overwrite outputs if they exist (ffmpeg -y). + -q, --quiet Reduce ffmpeg logging noise. + --lowpass FREQ Optional low-pass cutoff (e.g., 8000). Disabled by default. + --suffix SUF Suffix for output basename (default: _clean). + -h, --help Show this help. + +Notes: + - Default sample rate is 16 kHz mono PCM 16-bit (good for most speech ASR models). + - If arnndn (RNNoise) is used, it usually outperforms afftdn for speech denoise. + - The 'podcast' preset adds gentle dynamics and loudness normalization (single-pass). +EOF +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "Error: Required command '$1' not found in PATH" >&2 + exit 1 + } +} + +# Defaults +OUT_DIR="" +OUT_EXT="wav" +RN_MODEL="" +NO_ML=false +REQUIRE_ML=true # default: require RNNoise; install/guide if missing; fail fast if unavailable +PRESET="asr" +JOBS=1 +FORCE=false +QUIET=false +LOWPASS="" +SUFFIX="_clean" +HIGHPASS="80" +AFFTDN_NF="-25" # noise floor in dB for afftdn +AFFTDN_MD="8" # mode for afftdn (higher can be more aggressive); requires builds that support 'md' +NO_ADVANCED=false # when true, avoid advanced options that some ffmpeg builds lack + +# Parse args +if [[ $# -lt 1 ]]; then + print_usage + exit 1 +fi + +INPUT_PATH="$1"; shift || true + +while [[ $# -gt 0 ]]; do + case "$1" in + -O|--out-dir) + OUT_DIR="$2"; shift 2;; + -e|--ext) + OUT_EXT="$2"; shift 2;; + -m|--model) + RN_MODEL="$2"; shift 2;; + --no-ml) + NO_ML=true; shift;; + --preset) + PRESET="$2"; shift 2;; + -j|--jobs) + JOBS="$2"; shift 2;; + -f|--force) + FORCE=true; shift;; + -q|--quiet) + QUIET=true; shift;; + --lowpass) + LOWPASS="$2"; shift 2;; + --suffix) + SUFFIX="$2"; shift 2;; + --no-advanced|--compat) + NO_ADVANCED=true; shift;; + --allow-fallback) + REQUIRE_ML=false; shift;; + -h|--help) + print_usage; exit 0;; + *) + echo "Unknown option: $1" >&2 + print_usage + exit 1;; + esac +done + +require_cmd ffmpeg + +# Resolve FFmpeg binary (env override -> local build -> system) +FFMPEG_BIN=${FFMPEG_BIN:-} +if [[ -z "${FFMPEG_BIN}" ]]; then + if [[ -x "$SCRIPT_DIR/ffmpeg-build/FFmpeg/ffmpeg" ]]; then + FFMPEG_BIN="$SCRIPT_DIR/ffmpeg-build/FFmpeg/ffmpeg" + else + FFMPEG_BIN="ffmpeg" + fi +fi + +if ! command -v "$FFMPEG_BIN" >/dev/null 2>&1 && [[ ! -x "$FFMPEG_BIN" ]]; then + echo "Error: FFmpeg binary not found: $FFMPEG_BIN" >&2 + exit 1 +fi +if ! $QUIET; then + echo "Using FFmpeg binary: $FFMPEG_BIN" >&2 +fi + +FFMPEG_LOG=(-hide_banner) +if $QUIET; then + FFMPEG_LOG+=( -loglevel error ) +else + FFMPEG_LOG+=( -loglevel info ) +fi + +FFMPEG_OVERWRITE=(-n) +if $FORCE; then + FFMPEG_OVERWRITE=(-y) +fi + +arnndn_available=false +if "$FFMPEG_BIN" -hide_banner -h filter=arnndn >/dev/null 2>&1; then + arnndn_available=true +else + if "$FFMPEG_BIN" -hide_banner -filters 2>/dev/null | grep -Eq '(^|[[:space:]])arnndn([[:space:]]|$)'; then + arnndn_available=true + fi +fi +if ! $QUIET; then + echo "arnndn_available=$arnndn_available" >&2 +fi + +# Check if afftdn supports 'md' option +afftdn_supports_md=false +if "$FFMPEG_BIN" -hide_banner -h filter=afftdn 2>/dev/null | grep -q " md="; then + afftdn_supports_md=true +fi + +# Try to auto-discover an RNNoise model if none provided +find_default_rn_model() { + local candidate="" + # Allow env variable override + if [[ -n "${RNNOISE_MODEL:-}" && -f "${RNNOISE_MODEL}" ]]; then + echo "${RNNOISE_MODEL}" + return 0 + fi + local dirs=( + "$SCRIPT_DIR/models" + "$SCRIPT_DIR/../models" + "/usr/share/rnnoise" + "/usr/local/share/rnnoise" + "/usr/share/ffmpeg/models" + "$HOME/.local/share/rnnoise" + ) + # Prefer '.rnnn' models (rnnoise-nu style) over legacy '.nn' + local exts=("rnnn" "nn" "model") + for d in "${dirs[@]}"; do + if [[ -d "$d" ]]; then + for ext in "${exts[@]}"; do + # Pick the first matching model file + for f in "$d"/*."$ext"; do + if [[ -f "$f" ]]; then + echo "$f" + return 0 + fi + done + done + fi + done + return 1 +} + +use_arnndn=false +if [[ $NO_ML == false ]]; then + if [[ $arnndn_available == false ]]; then + if $REQUIRE_ML; then + echo "Error: FFmpeg 'arnndn' filter not available. Please install/upgrade FFmpeg with librnnoise (see Bash/install_ffmpeg_with_arnndn.sh)." >&2 + exit 9 + fi + else + # arnndn available; require an external model + if [[ -n "$RN_MODEL" && -f "$RN_MODEL" ]]; then + : + else + if model_path=$(find_default_rn_model); then + RN_MODEL="$model_path" + else + if [[ -x "$SCRIPT_DIR/get_rnnoise_model.sh" ]]; then + RN_TARGET_DIR="$SCRIPT_DIR/models" RN_TARGET_NAME="rnnoise_model.rnnn" "$SCRIPT_DIR/get_rnnoise_model.sh" --yes || true + if model_path=$(find_default_rn_model); then + RN_MODEL="$model_path" + fi + fi + fi + fi + if [[ -z "$RN_MODEL" ]]; then + echo "Error: RNNoise model required but not found. Automatic download failed." >&2 + echo "Hint: Set RN_URL to a reachable model URL and run Bash/get_rnnoise_model.sh, or supply -m /path/to/model.nn." >&2 + exit 10 + fi + use_arnndn=true + echo "Using RNNoise external model: $RN_MODEL" >&2 + fi +fi + +build_filters() { + local filters=() + # Remove low-frequency rumble typical for handheld/room noise + filters+=("highpass=f=${HIGHPASS}") + + # Denoise + if $use_arnndn; then + # arnndn with full mix keeps the model output; if no external model, rely on built-in + filters+=("aresample=48000") + filters+=("arnndn=m=${RN_MODEL}:mix=1.0") + else + # afftdn: FFT-based denoise, tune nf (noise floor) as needed + if $REQUIRE_ML; then + echo "Error: RNNoise required but not in use; aborting rather than falling back to afftdn. Use --allow-fallback to permit." >&2 + exit 11 + fi + if $NO_ADVANCED; then + filters+=("afftdn=nf=${AFFTDN_NF}") + else + if $afftdn_supports_md; then + filters+=("afftdn=nf=${AFFTDN_NF}:md=${AFFTDN_MD}") + else + echo "Error: Your ffmpeg's afftdn filter does not support 'md='." >&2 + echo "Hint: Install/upgrade ffmpeg to a build that supports afftdn md or rerun with --no-advanced." >&2 + echo " On Debian/Ubuntu you may need a newer ffmpeg from a PPA or build from source." >&2 + exit 8 + fi + fi + fi + + # Optional low-pass to shave hiss; keep disabled unless requested + if [[ -n "$LOWPASS" ]]; then + filters+=("lowpass=f=${LOWPASS}") + fi + + case "$PRESET" in + asr) + # ASR-friendly: avoid heavy gating/leveling, just prevent clipping + filters+=("alimiter=limit=0.94") + ;; + podcast) + # Gentle dynamic normalization and broadcast-ish loudness (single-pass) + # Note: single-pass loudnorm is approximate but OK for quick workflows + filters+=("dynaudnorm=f=500:g=5:p=0.1") + filters+=("loudnorm=i=-18:lra=9:tp=-2.0") + ;; + aggressive) + # Heavier clean-up; may harm ASR slightly but suppress background more + filters+=("agate=threshold=0.012:ratio=2.5:release=200") + filters+=("dynaudnorm=f=400:g=7:p=0.1") + filters+=("loudnorm=i=-18:lra=9:tp=-2.0") + ;; + *) ;; + esac + + # Resample and format at the end for ASR + filters+=("aresample=16000") + filters+=("aformat=channel_layouts=mono:sample_fmts=s16") + + local IFS=","; echo "${filters[*]}" +} + +make_out_path_for_file() { + local in_file="$1" + local base + base=$(basename -- "$in_file") + base="${base%.*}" + local out_base="${base}${SUFFIX}.${OUT_EXT}" + if [[ -n "$OUT_DIR" ]]; then + mkdir -p -- "$OUT_DIR" + echo "$OUT_DIR/$out_base" + else + local dir + dir=$(dirname -- "$in_file") + echo "$dir/$out_base" + fi +} + +process_one() { + local in_file="$1" + local out_file + out_file=$(make_out_path_for_file "$in_file") + + # Choose codec based on extension + local codec=( -c:a pcm_s16le ) + if [[ "$OUT_EXT" == "flac" ]]; then + codec=( -c:a flac ) + fi + + local af + af=$(build_filters) + + if [[ -f "$out_file" && $FORCE == false ]]; then + echo "Skip (exists): $out_file" + return 0 + fi + + echo "Cleaning: $in_file -> $out_file" + "$FFMPEG_BIN" "${FFMPEG_LOG[@]}" "${FFMPEG_OVERWRITE[@]}" -i "$in_file" -af "$af" "${codec[@]}" "$out_file" +} + +# Concurrency helpers (bash >= 5 supports wait -n; fallback to sequential if not) +supports_wait_n=false +if [[ -n "${BASH_VERSINFO:-}" && ${BASH_VERSINFO[0]} -ge 5 ]]; then + supports_wait_n=true +fi + +run_dir() { + local dir="$1" + # Common audio extensions (case-insensitive) + mapfile -d '' files < <(find "$dir" -type f \ + \( -iname "*.wav" -o -iname "*.mp3" -o -iname "*.m4a" -o -iname "*.aac" -o -iname "*.flac" \ + -o -iname "*.ogg" -o -iname "*.opus" -o -iname "*.wma" -o -iname "*.webm" \) -print0) + + if [[ ${#files[@]} -eq 0 ]]; then + echo "No audio files found in: $dir" + return 0 + fi + + local running=0 + for f in "${files[@]}"; do + if [[ "$JOBS" -le 1 || $supports_wait_n == false ]]; then + process_one "$f" + else + process_one "$f" & + ((running++)) + if (( running >= JOBS )); then + wait -n || true + ((running--)) + fi + fi + done + + # Wait for any remaining background jobs + if (( JOBS > 1 )) && $supports_wait_n; then + wait || true + fi +} + +main() { + # Sanity checks and notices + if [[ -n "$RN_MODEL" && $use_arnndn == false && $NO_ML == false ]]; then + echo "Note: arnndn filter not available in your ffmpeg or model missing — using afftdn." >&2 + fi + + if [[ -f "$INPUT_PATH" ]]; then + process_one "$INPUT_PATH" + elif [[ -d "$INPUT_PATH" ]]; then + run_dir "$INPUT_PATH" + else + echo "Error: Input path not found: $INPUT_PATH" >&2 + exit 1 + fi +} + +main "$@" diff --git a/get_rnnoise_model.sh b/get_rnnoise_model.sh new file mode 100755 index 0000000..2ac1057 --- /dev/null +++ b/get_rnnoise_model.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# get_rnnoise_model.sh — fetch an RNNoise model into a local models dir +# +# Prefers known-good rnnoise-nu models. You can override with: +# RN_URL, RN_TARGET_DIR, RN_TARGET_NAME +# +# Usage: +# Bash/get_rnnoise_model.sh # interactive download +# RN_TARGET_DIR=./models Bash/get_rnnoise_model.sh --yes + +ask_yes_no() { + read -r -p "$1 [y/N]: " ans || true + case "${ans:-}" in + y|Y|yes|YES) return 0;; + *) return 1;; + esac +} + +has_cmd() { command -v "$1" >/dev/null 2>&1; } + +YES=false +while [[ $# -gt 0 ]]; do + case "$1" in + -y|--yes) YES=true; shift;; + *) echo "Unknown option: $1" >&2; exit 2;; + esac +done + +RN_TARGET_DIR=${RN_TARGET_DIR:-"$(dirname "$0")/models"} +RN_TARGET_NAME=${RN_TARGET_NAME:-"rnnoise_model.rnnn"} + +mkdir -p "$RN_TARGET_DIR" +dest="$RN_TARGET_DIR/$RN_TARGET_NAME" + +if [[ -f "$dest" ]]; then + echo "Model already exists at: $dest" + exit 0 +fi + +if ! $YES; then + if ! ask_yes_no "Download RNNoise model to $dest?"; then + echo "Aborted." + exit 1 + fi +fi + +if ! has_cmd curl && ! has_cmd wget; then + echo "Error: Need curl or wget to download RNNoise model." >&2 + exit 3 +fi + +# Priority 1: explicit URL +if [[ -n "${RN_URL:-}" ]]; then + echo "Downloading RNNoise model from RN_URL: $RN_URL" >&2 + tmp=$(mktemp) + if has_cmd curl; then + curl -fsSL "$RN_URL" -o "$tmp" + else + wget -qO "$tmp" "$RN_URL" + fi + if [[ -s "$tmp" ]]; then + mv "$tmp" "$dest" + echo "Saved RNNoise model to: $dest" + exit 0 + fi + rm -f "$tmp" || true + echo "Warning: RN_URL download failed; continuing to fallback sources." >&2 +fi + +# Priority 2: rnnoise-nu known models (GregorR) +NU_URLS=( + "https://raw.githubusercontent.com/GregorR/rnnoise-nu/master/src/models/sh.rnnn" + "https://raw.githubusercontent.com/GregorR/rnnoise-nu/master/src/models/lq.rnnn" + "https://raw.githubusercontent.com/GregorR/rnnoise-nu/master/src/models/mp.rnnn" + "https://raw.githubusercontent.com/GregorR/rnnoise-nu/master/src/models/bd.rnnn" + "https://raw.githubusercontent.com/GregorR/rnnoise-nu/master/src/models/cb.rnnn" +) +for u in "${NU_URLS[@]}"; do + echo "Attempting to download RNNoise model from: $u" >&2 + tmp=$(mktemp) + if has_cmd curl; then + if curl -fsSL "$u" -o "$tmp"; then + if [[ -s "$tmp" ]]; then + mv "$tmp" "$dest" + echo "Saved RNNoise model to: $dest" >&2 + exit 0 + fi + fi + else + if wget -qO "$tmp" "$u"; then + if [[ -s "$tmp" ]]; then + mv "$tmp" "$dest" + echo "Saved RNNoise model to: $dest" >&2 + exit 0 + fi + fi + fi + rm -f "$tmp" || true +done + +# Priority 2b: arnndn-models fallback (richardpl) +RNNDN_URLS=( + "https://raw.githubusercontent.com/richardpl/arnndn-models/master/sh.rnnn" +) +for u in "${RNNDN_URLS[@]}"; do + echo "Attempting to download RNNoise model from: $u" >&2 + tmp=$(mktemp) + if has_cmd curl; then + if curl -fsSL "$u" -o "$tmp"; then + if [[ -s "$tmp" ]]; then + mv "$tmp" "$dest" + echo "Saved RNNoise model to: $dest" >&2 + exit 0 + fi + fi + else + if wget -qO "$tmp" "$u"; then + if [[ -s "$tmp" ]]; then + mv "$tmp" "$dest" + echo "Saved RNNoise model to: $dest" >&2 + exit 0 + fi + fi + fi + rm -f "$tmp" || true +done + +# Priority 3: repo archives (rnnoise-nu and arnndn-models) +ARCHIVES=( + "https://github.com/GregorR/rnnoise-nu/archive/refs/heads/master.zip" + "https://github.com/richardpl/arnndn-models/archive/refs/heads/master.zip" +) +for aurl in "${ARCHIVES[@]}"; do + echo "Attempting to download archive: $aurl" >&2 + tmpdir=$(mktemp -d) + archive="$tmpdir/models.zip" + set +e + if has_cmd curl; then + curl -fL "$aurl" -o "$archive" + else + wget -O "$archive" "$aurl" + fi + status=$? + set -e + if [[ $status -ne 0 ]]; then + rm -rf "$tmpdir" || true + continue + fi + if has_cmd bsdtar; then + bsdtar -xf "$archive" -C "$tmpdir" + elif has_cmd unzip; then + unzip -q "$archive" -d "$tmpdir" + else + echo "Warning: Need bsdtar or unzip to extract archive; skipping archive method." >&2 + rm -rf "$tmpdir" || true + continue + fi + mapfile -t nnfiles < <(bash -lc 'shopt -s globstar nullglob; for f in '"$tmpdir"'/**/*.rnnn '"$tmpdir"'/**/*.nn; do [[ -f "$f" ]] && echo "$f"; done') + if [[ ${#nnfiles[@]} -gt 0 ]]; then + cp -f "${nnfiles[0]}" "$dest" + echo "Saved RNNoise model to: $dest (from archive)" >&2 + rm -rf "$tmpdir" || true + exit 0 + fi + rm -rf "$tmpdir" || true +done + +# Priority 4: Arch-based AUR packages and search only .nn/.rnnn +if has_cmd yay; then + echo "Attempting to install AUR packages that may include RNNoise models..." >&2 + set +e + yay -S --noconfirm denoiseit-git 2>/dev/null + yay -S --noconfirm speech-denoiser-git 2>/dev/null + set -e + mapfile -t found < <(bash -lc 'shopt -s globstar nullglob; for f in /usr/share/**/*.nn /usr/share/**/*.rnnn /usr/local/share/**/*.nn /usr/local/share/**/*.rnnn; do [[ -f "$f" ]] && echo "$f"; done' 2>/dev/null || true) + if [[ ${#found[@]} -gt 0 ]]; then + echo "Found candidate models:" >&2 + printf ' %s\n' "${found[@]}" >&2 + cp -f "${found[0]}" "$dest" + echo "Copied model to: $dest" >&2 + exit 0 + fi +fi + +echo "Error: Could not obtain an RNNoise model automatically." >&2 +echo "Hint: Set RN_URL to a reachable model URL, or place a model file at: $dest" >&2 +exit 5 diff --git a/install_ffmpeg_with_arnndn.sh b/install_ffmpeg_with_arnndn.sh new file mode 100755 index 0000000..8c384c1 --- /dev/null +++ b/install_ffmpeg_with_arnndn.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# install_ffmpeg_with_arnndn.sh — helper to install/upgrade FFmpeg with arnndn and full audio filters +# +# Tries distro packages first; if not suitable, offers to build from source. +# This script prints commands and asks for confirmation before building. + +print_info() { + echo "[info] $*" +} + +ask_yes_no() { + read -r -p "$1 [y/N]: " ans || true + case "${ans:-}" in + y|Y|yes|YES) return 0;; + *) return 1;; + esac +} + +has_cmd() { command -v "$1" >/dev/null 2>&1; } + +detect_distro() { + if [[ -f /etc/os-release ]]; then + . /etc/os-release + echo "${ID:-unknown}" + else + echo "unknown" + fi +} + +main() { + local distro + distro=$(detect_distro) + print_info "Detected distro: $distro" + + if has_cmd ffmpeg && ffmpeg -hide_banner -filters | grep -q " arnndn "; then + print_info "Your ffmpeg already supports arnndn." + else + case "$distro" in + ubuntu|debian) + print_info "On Ubuntu/Debian, the official repo may lack newer filters. Consider a PPA or build from source." + echo "Options:" + echo " - ppa: sudo add-apt-repository ppa:savoury1/ffmpeg6 && sudo apt update && sudo apt install ffmpeg" + echo " - source build (recommended for latest): run this script to build from source" + ;; + arch|manjaro|endeavouros) + print_info "On Arch-based distros, ffmpeg is recent. Try: sudo pacman -Syu ffmpeg" + ;; + fedora) + print_info "On Fedora, try: sudo dnf install ffmpeg" + ;; + *) + print_info "Distro not recognized; will offer source build." + ;; + esac + fi + + if ask_yes_no "Build FFmpeg from source with rnnoise/arnndn support now?"; then + echo "This will clone FFmpeg and build locally under ./ffmpeg-build. Continue?" + if ! ask_yes_no "Proceed"; then + exit 0 + fi + set -x + mkdir -p ffmpeg-build && cd ffmpeg-build + # Prepare repository + if [[ -d FFmpeg ]]; then + if [[ -d FFmpeg/.git ]]; then + if ask_yes_no "An existing FFmpeg source directory was found. Reuse and update it?"; then + set +e + git -C FFmpeg fetch --all --tags --prune + git -C FFmpeg pull --rebase --ff-only || true + set -e + else + if ask_yes_no "Delete existing FFmpeg directory and re-clone?"; then + rm -rf FFmpeg + else + echo "Keeping existing FFmpeg directory as-is." + fi + fi + else + if ask_yes_no "Non-git 'FFmpeg' directory exists. Delete and re-clone?"; then + rm -rf FFmpeg + else + echo "Cannot proceed with a non-git FFmpeg directory present. Aborting." + exit 4 + fi + fi + fi + # Dependencies + if [[ "$distro" == "ubuntu" || "$distro" == "debian" ]]; then + sudo apt update + sudo apt install -y git build-essential yasm nasm pkg-config libx264-dev libx265-dev libvpx-dev libopus-dev libfdk-aac-dev libmp3lame-dev libvorbis-dev libass-dev libfreetype6-dev libgnutls28-dev libaom-dev libdav1d-dev libxvidcore-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev libxcb-shape0-dev libdrm-dev libvulkan-dev libva-dev libvdpau-dev librtmp-dev libunistring-dev libgnutls28-dev libchromaprint-dev libbluray-dev librubberband-dev libspeex-dev libsoxr-dev libvmaf-dev libzimg-dev libsvtav1-dev libtheora-dev libwebp-dev libopenal-dev libjack-jackd2-dev libpulse-dev librnnoise-dev + elif [[ "$distro" == "arch" || "$distro" == "manjaro" || "$distro" == "endeavouros" ]]; then + sudo pacman -Syu --needed base-devel yasm nasm pkgconf rnnoise + elif [[ "$distro" == "fedora" ]]; then + sudo dnf install -y git make gcc yasm nasm pkgconf-pkg-config rnnoise-devel libX11-devel libXext-devel libXfixes-devel libXv-devel libXrandr-devel libXi-devel libXtst-devel libXinerama-devel freetype-devel fontconfig-devel libass-devel libvpx-devel libaom-devel libdav1d-devel zimg-devel rubberband-devel soxr-devel libvorbis-devel opus-devel lame-devel + else + echo "Note: please ensure rnnoise development headers are installed (pkg-config rnnoise)." >&2 + fi + if [[ ! -d FFmpeg/.git ]]; then + git clone https://github.com/FFmpeg/FFmpeg.git --depth=1 + fi + cd FFmpeg + RN_FLAG="" + # Some FFmpeg versions auto-detect rnnoise without a flag; include the flag only if supported + if ./configure --help | grep -q "librnnoise"; then + RN_FLAG="--enable-librnnoise" + else + echo "[info] configure has no --enable-librnnoise; relying on auto-detection via pkg-config (rnnoise)." >&2 + fi + + ./configure \ + --enable-gpl --enable-nonfree \ + --enable-libx264 --enable-libx265 --enable-libvpx --enable-libopus --enable-libmp3lame \ + --enable-libvorbis --enable-libass --enable-fontconfig --enable-libfreetype \ + --enable-librubberband --enable-libsoxr --enable-libzimg --enable-libvmaf \ + --enable-libdav1d --enable-libaom --enable-libsvtav1 \ + ${RN_FLAG} \ + --enable-ffplay --enable-ffprobe + make -j"$(nproc)" + echo "Build complete. You can run ./ffmpeg-build/FFmpeg/ffmpeg from this folder or 'sudo make install' to install system-wide." + set +x + else + echo "Skipped building from source." + fi +} + +main "$@" From d2e750a20be3244dcdda0e2c62e3d1b6f2be1d68 Mon Sep 17 00:00:00 2001 From: Krzysztof kuhy Rudnicki Date: Sun, 26 Oct 2025 14:46:18 +0100 Subject: [PATCH 15/15] feat: add fix unity script and transwcibe --- fix_thorium_unity.sh | 144 +++++++++++++++++++++ fix_unity.sh | 289 +++++++++++++++++++++++++++++++++++++++++++ transcribe.sh | 0 3 files changed, 433 insertions(+) create mode 100644 fix_thorium_unity.sh create mode 100755 fix_unity.sh mode change 100644 => 100755 transcribe.sh diff --git a/fix_thorium_unity.sh b/fix_thorium_unity.sh new file mode 100644 index 0000000..8a2a5f0 --- /dev/null +++ b/fix_thorium_unity.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +# Configure Thorium/Chromium to auto-allow unityhub:// deep links from Unity login origins. +# This avoids missed external-protocol prompts and helps Unity Hub receive the token after web login. +# +# Features: +# - Install a system policy file (requires sudo) with AutoLaunchProtocolsFromOrigins for unityhub +# - Optionally set Thorium as default browser +# - Optionally restart Thorium +# - Non-destructive: does not edit your Thorium profile Preferences +# +# Usage: +# bash Bash/fix_thorium_unity.sh --policy # Install policy (sudo) +# bash Bash/fix_thorium_unity.sh --set-default # Set default browser to Thorium +# bash Bash/fix_thorium_unity.sh --restart # Restart Thorium +# bash Bash/fix_thorium_unity.sh --policy --restart # Install policy and restart browser + +set -euo pipefail +IFS=$'\n\t' + +GREEN="\033[1;32m"; YELLOW="\033[1;33m"; RED="\033[1;31m"; BLUE="\033[1;34m"; NC="\033[0m" +log_info() { echo -e "${BLUE}[INFO]${NC} $*"; } +log_ok() { echo -e "${GREEN}[ OK ]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERR ]${NC} $*" 1>&2; } + +DO_POLICY=false +SET_DEFAULT=false +DO_RESTART=false + +usage() { + cat </dev/null 2>&1; then + log_error "sudo not found; cannot install system policy. Use --set-default or run from root." + exit 1 + fi +} + +install_policy() { + ensure_sudo + # Candidate policy directories (most common for Chromium forks) + local candidates=( + "/etc/thorium-browser/policies/managed" # Thorium + "/etc/chromium/policies/managed" # Chromium + "/etc/opt/chrome/policies/managed" # Google Chrome + ) + local wrote_any=false + for target in "${candidates[@]}"; do + log_info "Installing policy into: $target" + sudo mkdir -p "$target" + local policy_file="$target/unityhub-policy.json" + sudo tee "$policy_file" >/dev/null <<'JSON' +{ + "AutoLaunchProtocolsFromOrigins": [ + { "protocol": "unityhub", "origin": "https://id.unity.com", "allow": true }, + { "protocol": "unityhub", "origin": "https://login.unity.com", "allow": true }, + { "protocol": "unityhub", "origin": "https://unity.com", "allow": true }, + { "protocol": "unity", "origin": "https://id.unity.com", "allow": true }, + { "protocol": "unity", "origin": "https://login.unity.com", "allow": true }, + { "protocol": "unity", "origin": "https://unity.com", "allow": true } + ] +} +JSON + # Some Chromium builds cache policies; no explicit reload on Linux. Restarting browser suffices. + log_ok "Policy written: $policy_file" + wrote_any=true + done + if [[ "$wrote_any" != true ]]; then + log_warn "Policy may not have been written. No candidate directories processed." + fi +} + +set_default_browser() { + if command -v xdg-settings >/dev/null 2>&1; then + # Prefer the upstream desktop id if it exists + local desktop="thorium-browser.desktop" + if [[ ! -f "/usr/share/applications/$desktop" && -f "$HOME/.local/share/applications/$desktop" ]]; then + : # keep desktop as is + elif [[ ! -f "/usr/share/applications/$desktop" && ! -f "$HOME/.local/share/applications/$desktop" ]]; then + log_warn "thorium-browser.desktop not found; leaving default browser unchanged." + return + fi + log_info "Setting default browser to $desktop" + xdg-settings set default-web-browser "$desktop" || log_warn "Failed to set default browser via xdg-settings" + log_ok "Default browser set to: $(xdg-settings get default-web-browser 2>/dev/null || echo "$desktop")" + else + log_warn "xdg-settings not found; cannot set default browser automatically." + fi +} + +restart_thorium() { + # Kill Thorium processes and start fresh + log_info "Restarting Thorium..." + pkill -9 -f 'thorium-browser' 2>/dev/null || true + # Also kill unityhub-bin's embedded Chromium if any leftover (harmless) + pkill -9 -f 'unityhub-bin' 2>/dev/null || true + # Start Thorium detached if available + if command -v thorium-browser >/dev/null 2>&1; then + nohup thorium-browser >/dev/null 2>&1 & disown || true + fi + log_ok "Thorium restart attempted." +} + +main() { + $DO_POLICY && install_policy + $SET_DEFAULT && set_default_browser + $DO_RESTART && restart_thorium + + cat <<'NEXT' +--- +Next steps: +- Open Unity Hub, click Sign in, complete in Thorium; when prompted, allow the unityhub link to open the app. +- If Thorium still does not prompt, the installed policy will auto-allow from Unity origins on next restart. +- You can also trigger a test link: xdg-open 'unityhub://v1/editor-signin' +--- +NEXT +} + +main "$@" diff --git a/fix_unity.sh b/fix_unity.sh new file mode 100755 index 0000000..ddd19d1 --- /dev/null +++ b/fix_unity.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash + +# Fix Unity Hub login on Linux (Arch/XFCE) by ensuring the unityhub:// URL scheme +# is correctly registered and handled. This script: +# - Detects Unity Hub installation type (Native, Flatpak, AppImage) +# - Creates a local desktop entry to handle x-scheme-handler/unityhub (and unity) +# - Registers the handler using xdg-mime and updates desktop database +# - Optionally installs required tools (xdg-utils, desktop-file-utils, portals) +# - Optionally tests the handler by opening a unityhub:// link +# +# Usage: +# bash Bash/fix_unity.sh # Run fix (no deps install, no test) +# bash Bash/fix_unity.sh -y # Auto-install deps (Arch) if missing +# bash Bash/fix_unity.sh --test # Also launches a test unityhub:// link +# bash Bash/fix_unity.sh -y --test # Install deps and run test +# +# Notes: +# - For Flatpak installs, Exec uses: flatpak run com.unity.UnityHub %U +# - For native installs, Exec uses the unityhub binary path with %U +# - Chromium/Thorium may prompt to "Open xdg-open" after web login—allow it. + +set -euo pipefail +IFS=$'\n\t' + +SCRIPT_NAME="$(basename "$0")" +GREEN="\033[1;32m"; YELLOW="\033[1;33m"; RED="\033[1;31m"; BLUE="\033[1;34m"; NC="\033[0m" + +log_info() { echo -e "${BLUE}[INFO]${NC} $*"; } +log_ok() { echo -e "${GREEN}[ OK ]${NC} $*"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +log_error() { echo -e "${RED}[ERR ]${NC} $*" 1>&2; } + +usage() { + cat </dev/null 2>&1; then + return 1 + fi +} + +ensure_deps_arch() { + # Best-effort install for Arch-based systems + if [[ "$AUTO_INSTALL" != true ]]; then + log_warn "Skipping package installation (use -y to auto-install)." + return 0 + fi + if ! require_cmd pacman; then + log_warn "Not an Arch-based system (pacman not found). Skipping auto-install." + return 0 + fi + local pkgs=(xdg-utils desktop-file-utils xdg-desktop-portal xdg-desktop-portal-gtk) + log_info "Installing/ensuring packages: ${pkgs[*]}" + if ! require_cmd sudo; then + log_warn "sudo not found; attempting pacman directly (may fail)." + sudo_cmd="" + else + sudo_cmd="sudo" + fi + # Use --needed to avoid reinstalling + set +e + $sudo_cmd pacman -S --needed --noconfirm "${pkgs[@]}" + local rc=$? + set -e + if [[ $rc -ne 0 ]]; then + log_warn "Package install may have failed or been partial. Continuing anyway." + else + log_ok "Dependencies installed/verified." + fi +} + +desktop_dir="$HOME/.local/share/applications" +mkdir -p "$desktop_dir" + +detect_unityhub() { + # Outputs: INSTALL_TYPE (FLATPAK|NATIVE|APPIMAGE|UNKNOWN) and EXEC_CMD + local install_type="UNKNOWN" exec_cmd="" + + # 1) Flatpak + if command -v flatpak >/dev/null 2>&1; then + if flatpak info com.unity.UnityHub >/dev/null 2>&1; then + install_type="FLATPAK" + exec_cmd="flatpak run com.unity.UnityHub %U" + echo "$install_type|$exec_cmd" + return 0 + fi + fi + + # 2) Native binary in PATH + if command -v unityhub >/dev/null 2>&1; then + local path + path="$(command -v unityhub)" + install_type="NATIVE" + exec_cmd="$path %U" + echo "$install_type|$exec_cmd" + return 0 + fi + + # 3) Search desktop files for Unity Hub Exec + local search_dirs=( + "$HOME/.local/share/applications" + "/usr/share/applications" + "/var/lib/flatpak/exports/share/applications" + "$HOME/.local/share/flatpak/exports/share/applications" + ) + local found_exec="" + for d in "${search_dirs[@]}"; do + [[ -d "$d" ]] || continue + # prefer official naming when present + local f + for f in "$d"/*.desktop; do + [[ -e "$f" ]] || continue + if grep -qiE '^(Name|Comment)=.*Unity Hub' "$f" 2>/dev/null || \ + grep -qiE 'Exec=.*unityhub' "$f" 2>/dev/null; then + local exec_line + exec_line="$(grep -iE '^Exec=' "$f" | head -n1 | sed 's/^Exec=//')" + if [[ -n "$exec_line" ]]; then + found_exec="$exec_line" + break 2 + fi + fi + done + done + + if [[ -n "$found_exec" ]]; then + # Normalize: ensure %U present + if [[ "$found_exec" != *"%U"* && "$found_exec" != *"%u"* ]]; then + found_exec+=" %U" + fi + if [[ "$found_exec" == flatpak* ]]; then + install_type="FLATPAK" + elif [[ "$found_exec" == *AppImage* || "$found_exec" == *appimage* ]]; then + install_type="APPIMAGE" + else + install_type="NATIVE" + fi + echo "$install_type|$found_exec" + return 0 + fi + + # 4) Try common AppImage locations + local ai_candidates=( + "$HOME/Applications/UnityHub*.AppImage" + "$HOME/.local/bin/UnityHub*.AppImage" + "/opt/UnityHub*/UnityHub*.AppImage" + ) + local ai + for ai in "${ai_candidates[@]}"; do + for p in $ai; do + if [[ -f "$p" && -x "$p" ]]; then + install_type="APPIMAGE" + exec_cmd="$p %U" + echo "$install_type|$exec_cmd" + return 0 + fi + done + done + + echo "$install_type|$exec_cmd" +} + +create_handler_desktop() { + local exec_cmd="$1" + local dest="$desktop_dir/unityhub-url-handler.desktop" + log_info "Writing handler desktop entry: $dest" + cat > "$dest" </dev/null 2>&1; then + update-desktop-database "$desktop_dir" || true + else + log_warn "update-desktop-database not found (install desktop-file-utils)." + fi + + # Register as default handler for both schemes + if command -v xdg-mime >/dev/null 2>&1; then + xdg-mime default "$(basename "$desktop_file")" x-scheme-handler/unityhub || true + xdg-mime default "$(basename "$desktop_file")" x-scheme-handler/unity || true + else + log_error "xdg-mime not found (install xdg-utils)." + return 1 + fi + log_ok "MIME handler registered for unityhub:// (and unity://)." +} + +verify_registration() { + local expected="$(basename "$1")" + local cur1="$(xdg-mime query default x-scheme-handler/unityhub 2>/dev/null || true)" + local cur2="$(xdg-mime query default x-scheme-handler/unity 2>/dev/null || true)" + log_info "Current handler (unityhub): ${cur1:-}" + log_info "Current handler (unity): ${cur2:-}" + if [[ "$cur1" == "$expected" ]]; then + log_ok "unityhub scheme correctly set to $expected" + else + log_warn "unityhub scheme not set to $expected (currently: ${cur1:-none})." + fi +} + +maybe_test_open() { + if [[ "$RUN_TEST" == true ]]; then + log_info "Opening test link: unityhub://v1/editor-signin" + if command -v xdg-open >/dev/null 2>&1; then + xdg-open 'unityhub://v1/editor-signin' >/dev/null 2>&1 || true + log_ok "Test link invoked. Check if Unity Hub launches or focuses." + else + log_warn "xdg-open not found; cannot run test automatically." + fi + else + log_info "You can test manually with: xdg-open 'unityhub://v1/editor-signin'" + fi +} + +main() { + log_info "Ensuring required tools (optional)." + ensure_deps_arch + + log_info "Detecting Unity Hub installation..." + IFS='|' read -r install_type exec_cmd < <(detect_unityhub) + log_info "Detected type: $install_type" + if [[ -z "${exec_cmd:-}" ]]; then + log_warn "Could not find Unity Hub executable automatically." + log_warn "- If using Flatpak: install with 'flatpak install flathub com.unity.UnityHub'" + log_warn "- If native (AUR): ensure 'unityhub' is in PATH" + log_warn "- If AppImage: place it in ~/Applications and make it executable" + log_error "Aborting—no Exec command available to create handler." + exit 2 + fi + log_info "Using Exec: $exec_cmd" + + local desktop_file + desktop_file="$(create_handler_desktop "$exec_cmd")" + + register_mime_handler "$desktop_file" + verify_registration "$desktop_file" + + cat <<'NOTE' +--- +Next steps: +- Sign in from Unity Hub. When the browser finishes, ALLOW the prompt to open xdg-open/Unity Hub. +- If Thorium suppresses the external protocol prompt, try once with Firefox/Chromium to confirm. +--- +NOTE + + maybe_test_open + + log_ok "Done. If login still fails, check the Hub's logs and share the outputs of:\n which unityhub || true\n flatpak info com.unity.UnityHub 2>/dev/null | sed -n '1,5p' || true\n xdg-mime query default x-scheme-handler/unityhub\n grep -R "x-scheme-handler/unityhub" ~/.local/share/applications /usr/share/applications 2>/dev/null | head -n 10" +} + +main "$@" diff --git a/transcribe.sh b/transcribe.sh old mode 100644 new mode 100755