#! /bin/sh -e # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2025 Munyoki Kilyungi # # This file is part of genenetwork-machines. # # genenetwork-machines is free software: you can redistribute it # and/or modify it under the terms of the GNU General Public License # as published by the Free Software Foundation, either version 3 of # the License, or (at your option) any later version. # # genenetwork-machines is distributed in the hope that it will be # useful, but WITHOUT ANY WARRANTY; without even the implied warranty # of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with genenetwork-machines. If not, see # . # Build and install genenetwork container on your local machine. set -euo pipefail BASE_DIR="${HOME:-/home/$USER}/genenetwork" SYSTEM_DIRECTORIES=( "$BASE_DIR/var/log" "$BASE_DIR/var/genenetwork" "$BASE_DIR/etc/genenetwork/conf" "$BASE_DIR/etc/genenetwork" "$BASE_DIR/var/lib/redis" "$BASE_DIR/var/lib/virtuoso" "$BASE_DIR/var/lib/xapian" "$BASE_DIR/var/genenetwork/genotype-files/genotype/json" "$BASE_DIR/var/lib/genenetwork-sqlite" "$BASE_DIR/var/lib/genenetwork-gnqa" "/tmp/local-container" ) GN_PROJECTS=( "genenetwork2:git@github.com:genenetwork/genenetwork2" "genenetwork3:git@github.com:genenetwork/genenetwork3" "gn-auth:https://git.genenetwork.org/gn-auth" "gn-docs:https://git.genenetwork.org/gn-docs" ) # File mappings: source -> destination declare -A FILE_MAPPINGS=( ["etc/auth.db"]="$BASE_DIR/var/genenetwork/auth.db" ["etc/llm.db"]="$BASE_DIR/var/lib/genenetwork-sqlite/llm.db" ["etc/gn2-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn2/secrets.py" ["etc/gn3-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn3/secrets.py" ["etc/gn-auth-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn-auth/secrets.py" ) CONTAINER_SCM="genenetwork-local-container.scm" CONTAINER_BIN="/usr/local/bin/genenetwork-local-container" GC_ROOT="/var/guix/gcroots/genenetwork-local-container" log() { local level="$1" shift echo "[$(date '+%Y-%m-%d %H:%M:%S')] $level: $*" >&2 } # Check dependencies for cmd in git guix sudo diff cp grep; do if ! command -v "$cmd" &>/dev/null; then log "ERROR" "Required command '$cmd' not found" exit 1 fi done # Check for gn-bioinformatics channel guix describe | grep gn-bioinformatics &> /dev/null && log "INFO" "guix guix-informatics $(guix describe | grep gn-bioinformatics | cut -d ' ' -f 4)" || (log "ERROR" "Please make sure your current profile has gn-bioinformatics" && exit 1) # Validate HOME is set if [ -z "${HOME:-}" ]; then log "ERROR" "HOME environment variable is not set" exit 1 fi if [ "$1" = "--init-container" ]; then log "INFO" "Creating system directories..." for dir in "${SYSTEM_DIRECTORIES[@]}"; do # Check if directory exists and is accessible if [ -d "$dir" ]; then if [ -w "$dir" ]; then log "DEBUG" "Directory exists and is writable: $dir" else log "WARNING" "Directory exists but is not writable: $dir. Making this writable" sudo chown -R "$USER" "$dir" fi continue fi # Attempt to create directory log "INFO" "Creating directory: $dir" if [ -w "$(dirname "$dir")" ]; then # Parent directory is writable, try without sudo if ! mkdir -p "$dir"; then log "ERROR" "Failed to create directory without sudo: $dir" exit 1 fi else # Parent directory requires root, use sudo if ! sudo mkdir -p "$dir"; then log "ERROR" "Failed to create directory with sudo: $dir" exit 1 fi # Set ownership to current user if created with sudo if ! sudo chown "$USER:$USER" "$dir"; then log "WARNING" "Failed to set ownership for: $dir" fi fi done # Check and copy configuration files log "INFO" "Checking and copying configuration files..." for src in "${!FILE_MAPPINGS[@]}"; do dest="${FILE_MAPPINGS[$src]}" log "INFO" "Processing $src -> $dest" # Check if source file exists if [ ! -f "$src" ]; then log "ERROR" "Source file does not exist: $src" exit 1 fi # Check if destination file exists if [ ! -f "$dest" ]; then log "INFO" "Destination file does not exist, copying $src to $dest" mkdir -p "$(dirname "$dest")" if ! cp "$src" "$dest"; then log "ERROR" "Failed to copy $src to $dest" exit 1 fi continue fi # Compare files using diff log "INFO" "Comparing $src with $dest" if diff_output=$(diff -u "$dest" "$src" 2>&1); then log "INFO" "Files $src and $dest are identical" else log "INFO" "Differences found between $src and $dest:" echo "$diff_output" >&2 log "INFO" "Copying $src to $dest" if ! cp "$src" "$dest"; then log "ERROR" "Failed to copy $src to $dest" exit 1 fi fi done is_git_repository() { local dir="$1" # Check for standard repository if [ -d "$dir/.git" ]; then if [ -f "$dir/.git/HEAD" ] && [ -d "$dir/.git/refs" ]; then log "DEBUG" "Detected standard Git repository: $dir" return 0 else log "ERROR" "Directory $dir/.git exists but is not a valid Git repository" return 1 fi # Check for bare repository elif [ -f "$dir/HEAD" ] && [ -d "$dir/refs" ] && [ -d "$dir/objects" ]; then log "DEBUG" "Detected bare Git repository: $dir" return 0 else log "ERROR" "$dir exists but is not a Git repository (neither standard nor bare)" return 1 fi } # Clone GeneNetwork projects log "INFO" "Cloning GeneNetwork projects..." for project_entry in "${GN_PROJECTS[@]}"; do IFS=':' read -r project repo_url <<< "$project_entry" dir="$BASE_DIR/$project" if [ ! -d "$dir" ]; then log "INFO" "Cloning $project from $repo_url to $dir" if ! git clone "$repo_url" "$dir"; then log "ERROR" "Failed to clone $project" exit 1 fi else log "DEBUG" "Directory exists, skipping clone: $dir" if ! is_git_repository "$dir"; then log "ERROR" "$dir exists but is not a Git repository" exit 1 fi fi done FLASK_SESSION="$BASE_DIR/genenetwork2/flask_session" log "INFO" "Checking FLASK_SESSION directory: $FLASK_SESSION" if [ ! -d "$FLASK_SESSION" ]; then log "INFO" "Creating FLASK_SESSION directory: $FLASK_SESSION" if ! mkdir -p "$FLASK_SESSION"; then log "ERROR" "Failed to create FLASK_SESSION directory: $FLASK_SESSION" exit 1 fi else log "DEBUG" "FLASK_SESSION directory already exists: $FLASK_SESSION" fi # Verify container SCM file exists if [ ! -f "$CONTAINER_SCM" ]; then log "ERROR" "Container SCM file not found: $CONTAINER_SCM" exit 1 fi # Create Guix system container log "INFO" "Creating Guix system container..." SHARE_OPTS=( "--share=$BASE_DIR/var/log=/var/log" "--share=$BASE_DIR/var/genenetwork=/var/genenetwork" "--share=$BASE_DIR/etc/genenetwork/conf=/etc/genenetwork/conf" "--share=$BASE_DIR/etc/genenetwork=/etc/genenetwork" "--share=$BASE_DIR/var/lib/redis=/var/lib/redis" "--share=$BASE_DIR/var/lib/virtuoso=/var/lib/virtuoso" "--share=$BASE_DIR/genenetwork2=/genenetwork2" "--share=$BASE_DIR/genenetwork3=/genenetwork3" "--share=$BASE_DIR/gn-auth=/gn-auth" "--share=$BASE_DIR/var/lib/xapian=/var/lib/xapian" "--share=$BASE_DIR/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite" "--share=$BASE_DIR/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa" "--share=/tmp/local-container=/tmp" "--share=$BASE_DIR/gn-docs=/var/lib/gn-docs" "--share=/run/mysqld=/run/mysqld" ) container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ "${SHARE_OPTS[@]}" \ "$CONTAINER_SCM") log $container_script # Create symbolic links log "INFO" "Creating symbolic links..." if ! sudo ln -sf "$container_script" "$CONTAINER_BIN"; then log "ERROR" "Failed to create symbolic link: $CONTAINER_BIN" exit 1 fi if ! sudo ln -sf "$container_script" "$GC_ROOT"; then log "ERROR" "Failed to create GC root link: $GC_ROOT" exit 1 fi log "INFO" "Setup completed successfully!" log "INFO" "Container script: $container_script" log "INFO" "Run with: $CONTAINER_BIN" log "INFO" "Email: test@development.user" log "INFO" "Password: testpasswd" fi if [ "$1" = "--init-sql" ]; then # Configuration URL="https://files.genenetwork.org/database/db_webqtl_s-2025-02-18.sql.xz" DOWNLOAD_DIR="/tmp" FILE_NAME=$(basename "$URL") EXTRACTED_FILE="${FILE_NAME%.xz}" DB_USER="webqtlout" DB_PASSWORD="webqtlout" DB_HOST="localhost" DB_NAME="db_webqtl_local" MYSQL_ROOT_USER="" # Change to your MySQL admin user if different MYSQL_ROOT_PASSWORD="" # Set this or leave empty to prompt # Check for required tools for cmd in wget xz mysql; do if ! command -v "$cmd" &> /dev/null; then log ERROR "Required command '$cmd' not found" exit 1 fi done # Download the file log INFO "Downloading $URL to $DOWNLOAD_DIR/$FILE_NAME" if ! wget -O "$DOWNLOAD_DIR/$FILE_NAME" "$URL"; then log ERROR "Failed to download $URL" exit 1 fi # Extract the .xz file log INFO "Extracting $DOWNLOAD_DIR/$FILE_NAME" if ! xz -d "$DOWNLOAD_DIR/$FILE_NAME"; then log ERROR "Failed to extract $DOWNLOAD_DIR/$FILE_NAME" rm -f "$DOWNLOAD_DIR/$FILE_NAME" exit 1 fi # Prepare MySQL user and password credentials if [ -z "$MYSQL_ROOT_USER" ]; then log INFO "MySQL root user not set, prompting for input" read -s -p "Enter MySQL user: " MYSQL_ROOT_USER fi if [ -z "$MYSQL_ROOT_PASSWORD" ]; then log INFO "MySQL root password not set, prompting for input" read -s -p "Enter MySQL root password: " MYSQL_ROOT_PASSWORD fi # Check if DB user exists, create if not log INFO "Checking if MySQL user $DB_USER exists" USER_EXISTS=$(mysql -h "$DB_HOST" -u "$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = '$DB_USER' AND host = 'localhost') AS user_exists;" 2>/dev/null | grep -o '[0-1]$') if [ "$USER_EXISTS" = "0" ]; then log INFO "Creating MySQL user $DB_USER" if ! mysql -h "$DB_HOST" -u "$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "CREATE USER '$DB_USER'@'localhost' IDENTIFIED BY '$DB_PASSWORD'; GRANT ALL PRIVILEGES ON $DB_NAME.* TO '$DB_USER'@'localhost'; FLUSH PRIVILEGES;" 2>/dev/null; then log ERROR "Failed to create MySQL user $DB_USER" rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" exit 1 fi else log INFO "User $DB_USER already exists, ensuring privileges" if ! mysql -h "$DB_HOST" -u "$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "GRANT ALL PRIVILEGES ON $DB_NAME.* TO '$DB_USER'@'localhost'; FLUSH PRIVILEGES;" 2>/dev/null; then log ERROR "Failed to update privileges for $DB_USER" rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" exit 1 fi fi # Create database if it doesn't exist log INFO "Ensuring database $DB_NAME exists" if ! mysql -h "$DB_HOST" -u "$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS $DB_NAME;" 2>/dev/null; then log ERROR "Failed to create or verify database $DB_NAME" rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" exit 1 fi # Install the SQL dump into the database log INFO "Importing $DOWNLOAD_DIR/$EXTRACTED_FILE into $DB_NAME" if ! mysql -h "$DB_HOST" -u "$DB_USER" -p"$DB_PASSWORD" "$DB_NAME" < "$DOWNLOAD_DIR/$EXTRACTED_FILE"; then log ERROR "Failed to import $DOWNLOAD_DIR/$EXTRACTED_FILE into $DB_NAME" rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" exit 1 fi # Clean up log INFO "Removing $DOWNLOAD_DIR/$EXTRACTED_FILE" rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" log INFO "Database import completed successfully" exit 0 fi