Alter README with current versions statuses

.gitignore

@@ -1,2 +0,0 @@
-variables.conf
-build/*

README.md

@@ -1,19 +1,14 @@
 # ArchRepo
-## Version 1.1
+## Version 2
 
-### Version 1
-In a nutshell:
-* this script is run punctually, on a dedicated Arch VM;
-* Process:
-  * it takes all packages in the repository;
-  * checks online if a new version is available
-  * if so build it and then add it to the repository
+### Past versions
+* [Version 1](https://gitea.kto.black/alex/archrepo/src/branch/version1) is the previous version, running state-fully in
+a dedicated VM, which uses NFS to mount the repository (and thus works only on a local network)
+  * [Version 1.1](https://gitea.kto.black/alex/archrepo/src/branch/version1.1), currently used, is an adapted version 1
+  with support for a remote repository (via SCP)
 
-### Version 1.1
-Version 1.1 is an adapted version 1, with remote repository. The idea is to replace NFS by SCP for everything related
-with connection to repository.
-
-We also have a specific clearing script, which should run directly on remote server.
-
-#### TODO
-* Replace `scp` calls by `sftp` (or `rsync`)
+### Introduction
+Version 2 targets an all-new architecture:
+* written in Python
+* running in a container
+  * potentially a stateless and ephemeral container

clear-repo.sh

@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-
-# The purpose of this script is to clear our custom pacman repository
-# i.e. remove old versions of packages
-# This script should NOT be run as root.
-
-# Make sure that AWK rules file is present, readable and correctly set in this script's parameters
-
-init() {
-  source variables.conf
-  repo_db_file=$repo_name.db.tar
-  repo_db_filesfile=$repo_name.files.tar
-  awk_rules_file='parser.awk'
-  if [ ! -e $repo_directory/$repo_db_file ] ; then
-    echo "Could not find db file. Exiting..."
-    exit 1
-  fi
-  echo "ArchRepo clearing script"
-}
-
-clearRepo() {
-  if [ "$(uname)" = "OpenBSD" ]; then # TODO This could probably be done better (like with actually checking if tar is GNU...)
-    gnutar="gtar"
-  else
-    gnutar="tar"
-  fi
-  stream=$($gnutar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
-
-  OIFS=$IFS
-  IFS='|'
-  stream="${stream:1}"
-  current=""
-  for package in $stream
-  do
-    IFS='
-'
-    count=0
-    for line in $package; do
-      if [ $count -eq 0 ]; then
-        current="$current $line"
-      else
-        continue
-      fi
-
-      ((count++))
-    done
-
-    IFS='|'
-  done
-  IFS=$OIFS
-
-  cd $repo_directory
-  reg="^($repo_name.).+"
-  for filename in *; do
-    if [[ $filename =~ $reg || $filename = 'sync' || $filename = 'local'  || $filename = 'resources' ]]; then
-      continue
-    fi
-    for package in $current; do
-      if [ $filename = $package ]; then
-        continue 2
-      fi
-    done
-    echo "Removing $filename..."
-    rm $filename
-  done
-}
-
-run_directory_path=$(pwd)
-
-# Main process
-
-if [ "$1" = "" ]; then
-  init
-  clearRepo
-else
-  echo "Please call this script without any arguments."
-  exit 1
-fi
-
-exit 0

parser.awk

@@ -1,17 +0,0 @@
-BEGIN {
-  printit = 0
-}
-
-{
-  if (printit == 1) {
-    print
-    printit = 0
-  }
-  if (/%FILENAME%/) {
-    printf "|"
-    printit = 1
-  }
-  if (/%NAME%/ || /%VERSION%/) {
-    printit = 1
-  }
-}

update-repo.py

@@ -0,0 +1,80 @@
+# KTO ArchRepo update script
+# version 2.0
+
+# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
+# provide already built packages of AUR software.
+
+# This version 2 aims at 2 major evolutions:
+# * manage dependencies (hence Python and object-oriented programming);
+# * run this script in a stateless Arch container, rather than a full VM as in version 1.
+# Additionaly, we may like to manage working directory in a better way (check available storage space, and existing
+# directories).
+
+# This script should NOT be run as root. Or should it?
+# * We are in a stateless container, but I don't think we can run makepkg as root...
+
+# Imports
+import os
+
+# Class for AUR package
+class Package:
+  def __init__(self, name):
+    self.name = name
+    # + version, dependencies (which should contain version as well)
+
+def parseDb(db_path):
+  # here we should use AWK to parse our repo database file and populate the collection a collection of packages to build
+  # that we then return
+
+def checkUpdates(packages_to_build): # or checkDependencies?
+  # here we should check for all packages in packages_to_build if they need an update and:
+  # * remove those which do not from the collection
+  # * check (recursively) for all these packages_to_build if they have dependencies
+  #   * if AUR dependencies, add them to this collection
+  #   * if Arch dependencies, add them to another collection arch_packages_to_install
+
+def installArchPackages(packages):
+  # packages is a collection of packages (objects or names?) to install
+  os.system("sudo pacman -Syu " + packages.toSpaceString()) # [].toSpaceString() being to define...
+
+def buildAurPackage(package):
+  # Fetch package (git clone)
+  # cd to directory
+  os.system("makepkg -s --no-confirm --no-progress-bar")
+
+def addPackageToRepo(package, repo):
+  os.system("cp...")
+  os.system("repo-add...")
+
+def sortByDependencies():
+  # packages_to_build should be sorted by dependencies here
+  # packages that are dependencies should be tagged adequately
+
+def init():
+  # Check for main dependencies:
+  deps = ['git', 'awk', 'package-query']
+  installArchPackages(deps)
+
+# This function is a "main" function
+def updateRepo():
+  init()
+  packages_to_build = parseDb(db_path)
+  checkUpdates() # ideally, this should directly edit packages_to_build AND arch_packages_to_install (I believe we can return tuples in Python)
+  installArchPackages(arch_packages_to_install)
+  sortByDependencies()
+  for package in packages_to_build:
+    buildAurPackage(package)
+    addPackageToRepo(package, repo)
+    if package.isDependency:
+      installArchPackages([package])
+
+def newPackage(package):
+  init()
+  checkUpdates([package])
+  installArchPackages(arch_packages_to_install)
+  sortByDependencies()
+  for package in packages_to_build:
+    buildAurPackage(package)
+    addPackageToRepo(package, repo)
+    if package.isDependency:
+      installArchPackages([package])

update-repo.sh

@@ -1,226 +0,0 @@
-#!/usr/bin/env bash
-
-# The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
-# provide package-ready AUR softs.
-# This script should NOT be run as root.
-
-# Dependencies
-# Make sure that AWK rules file is present, readable and correctly set in this script's parameters
-# Make sure the following packages are installed before running this script:
-# package-query aurutils git
-
-
-# Functions
-
-##################################################################
-ascii_frag() {
-    expr match "$1" "\([^[:digit:]]*\)"
-}
-
-ascii_remainder() {
-    expr match "$1" "[^[:digit:]]*\(.*\)"
-}
-
-numeric_frag() {
-    expr match "$1" "\([[:digit:]]*\)"
-}
-
-numeric_remainder() {
-    expr match "$1" "[[:digit:]]*\(.*\)"
-}
-
-# return 1 for $1 > $2
-# return 2 for $1 < $2
-# return 0 for equal
-vercomp() {
-    local WORK1="$1"
-    local WORK2="$2"
-    local NUM1="", NUM2="", ASCII1="", ASCII2=""
-    while true; do
-        ASCII1=`ascii_frag "${WORK1}"`
-        ASCII2=`ascii_frag "${WORK2}"`
-        WORK1=`ascii_remainder "${WORK1}"`
-        WORK2=`ascii_remainder "${WORK2}"`
-
-        if [ "${ASCII1}" \> "${ASCII2}" ]; then
-            return 1
-        elif [ "${ASCII1}" \< "${ASCII2}" ]; then
-            return 2
-        fi
-
-        NUM1=`numeric_frag "${WORK1}"`
-        NUM2=`numeric_frag "${WORK2}"`
-        WORK1=`numeric_remainder "${WORK1}"`
-        WORK2=`numeric_remainder "${WORK2}"`
-
-        if [ -z "${NUM1}" -a -z "${NUM2}" ]; then
-            return 0
-        elif [ -z "${NUM1}" -a -n "${NUM2}" ]; then
-            return 2
-        elif [ -n "${NUM1}" -a -z "${NUM2}" ]; then
-            return 1
-        fi
-
-        if [ "${NUM1}" -gt "${NUM2}" ]; then
-            return 1
-        elif [ "${NUM1}" -lt "${NUM2}" ]; then
-            return 2
-        fi
-    done
-}
-##################################################################
-
-init() {
-  # Only run if the user is not root
-  if [[ $USER = 'root' ]] ; then
-    echo "You cannot run this script as root!"
-    exit 1
-  fi
-
-  if [ -n "$(ls $run_directory_path)" ] ; then
-    echo "The running directory is not empty!"
-    exit 1
-  fi
-
-  source ../variables.conf
-  repo_db_file=$repo_name.db.tar
-  repo_db_filesfile=$repo_name.files.tar
-  awk_rules_file='../parser.awk'
-  eval "$(ssh-agent)"
-  ssh-add $ssh_key
-
-  # TODO Check if required packages are installed
-
-  echo "ArchRepo update script"
-}
-
-checkUpdates() {
-  echo "Checking for updates:"
-  upgraded_packages=""
-  scp $ssh_options $remote_repository/$repo_db_file $remote_repository/$repo_db_filesfile $run_directory_path/
-  stream=$(tar xOf $run_directory_path/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
-
-  OIFS=$IFS
-  IFS='|'
-  stream="${stream:1}"
-  for package in $stream
-  do
-    IFS='
-'
-    count=0
-    for line in $package; do
-      if [ $count -eq 0 ]; then
-        filename=$line
-      elif [ $count -eq 1 ]; then
-        name=$line
-      elif [ $count -eq 2 ]; then
-        version=$line
-      fi
-
-      ((count++))
-    done
-
-    reg="(linux-lts).+-((docs)|(headers))"
-    if [[ $name =~ $reg ]]; then
-      echo -e "\nSkipping $name ($version, default exception)."
-      continue 1
-    fi
-
-    reg="(kodi-addon-game).+-((debug))"
-    if [[ $name =~ $reg ]]; then
-      echo -e "\nSkipping $name ($version, default exception)."
-      continue 1
-    fi
-
-    if echo $exlist | grep -w $name > /dev/null; then
-      echo -e "\nSkipping $name ($version, temporary exception)."
-      continue 1
-    fi
-
-    reg="(package-query-debug)"
-    if [[ $name =~ $reg ]]; then
-      echo -e "\nSkipping $name ($version, default exception)."
-      continue 1
-    fi
-
-    echo -e "\nChecking $name ($version)..."
-    aur_version=$(package-query -A -f %v $name)
-
-    vercomp $aur_version $version
-    result=$?
-    if [[ $result -eq 1 ]]; then
-      echo "New version for $name: $version -> $aur_version"
-      IFS=$OIFS
-      createPackage $name
-      addPackageToRepo $name $aur_version
-      upgraded_packages="$upgraded_packages $name"
-    fi
-
-    IFS='|'
-  done
-  IFS=$OIFS
-  echo "All packages processed."
-
-  if [[ $upgraded_packages != "" ]]; then
-    echo "The following packages were upgraded:"
-    echo $upgraded_packages
-  fi
-}
-
-addPackage() {
-  createPackage $1
-  addPackageToRepo $1
-}
-
-createPackage() { # Create package $1
-  echo "Fetching $1 from AUR..."
-  aur fetch $1 # TODO This could be replaced by a git clone or even a wget+tar thanks to package-query
-  echo "Making $1 package..."
-  cd $1
-  makepkg -s --noconfirm --noprogressbar
-  cd $run_directory_path
-}
-
-# TODO: try manually: can we update db file before sending to remote repo?
-addPackageToRepo() { # Add package $1
-  echo "Adding $1 to repository..."
-  scp $ssh_options $run_directory_path/$1/$1-$2*.pkg.tar.* $remote_repository/
-  repo-add $repo_db_file $run_directory_path/$1/$1-$2*.pkg.tar.*
-  reg="(linux-lts).+"
-  if [[ $name =~ $reg ]]; then
-    echo "Adding $1-headers and $1-docs to repository..."
-    repo-add $repo_db_file $run_directory_path/$1/$1-headers-$2*.pkg.tar.* $run_directory_path/$1/$1-docs-$2*.pkg.tar.*
-    scp $ssh_options $run_directory_path/$1/$1-headers-$2*.pkg.tar.* $run_directory_path/$1/$1-docs-$2*.pkg.tar.* $remote_repository/
-  fi
-  scp $ssh_options $run_directory_path/$repo_db_file $run_directory_path/$repo_db_file.old $run_directory_path/$repo_db_filesfile $run_directory_path/$repo_db_filesfile.old $remote_repository/
-}
-
-clean() {
-  echo "Cleaning directory..."
-  rm -rf $run_directory_path/*
-  echo "Done."
-}
-
-run_directory_path=$(pwd)
-
-# Main process
-
-if [ "$1" = "" ]; then
-  init
-  checkUpdates
-elif [ "$1" = "add" ]; then
-  init
-  scp $ssh_options $remote_repository/$repo_db_file $remote_repository/$repo_db_filesfile $run_directory_path/
-  for arg in "$@"; do
-    if [[ $arg = "add" ]]; then
-      continue
-    fi
-    addPackage $arg
-  done
-elif [ "$1" = "clean" ]; then
-  clean
-else
-  exit 1
-fi
-
-exit 0

variables.conf.template

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-# Copy this file as variables.conf, with the correct values
-
-# For both update and clear scripts:
-repo_name=""
-
-# For update script:
-remote_repository=""
-ssh_key=""
-ssh_options="-i $ssh_key"
-exlist=""
-
-# For clear script:
-repo_directory=""