Add conf file, gitignore and move exlist to conf file

.gitignore

@@ -1,2 +1 @@
 variables.conf
-build/*

README.md

@@ -1,19 +1,10 @@
 # ArchRepo
-## Version 1.1
+## Version 1
 
-### Version 1
+### Introduction
 In a nutshell:
 * this script is run punctually, on a dedicated Arch VM;
 * Process:
   * it takes all packages in the repository;
   * checks online if a new version is available
   * if so build it and then add it to the repository
-
-### Version 1.1
-Version 1.1 is an adapted version 1, with remote repository. The idea is to replace NFS by SCP for everything related
-with connection to repository.
-
-We also have a specific clearing script, which should run directly on remote server.
-
-#### TODO
-* Replace `scp` calls by `sftp` (or `rsync`)

clear-repo.sh

@@ -1,80 +0,0 @@
-#!/usr/bin/env bash
-
-# The purpose of this script is to clear our custom pacman repository
-# i.e. remove old versions of packages
-# This script should NOT be run as root.
-
-# Make sure that AWK rules file is present, readable and correctly set in this script's parameters
-
-init() {
-  source variables.conf
-  repo_db_file=$repo_name.db.tar
-  repo_db_filesfile=$repo_name.files.tar
-  awk_rules_file='parser.awk'
-  if [ ! -e $repo_directory/$repo_db_file ] ; then
-    echo "Could not find db file. Exiting..."
-    exit 1
-  fi
-  echo "ArchRepo clearing script"
-}
-
-clearRepo() {
-  if [ "$(uname)" = "OpenBSD" ]; then # TODO This could probably be done better (like with actually checking if tar is GNU...)
-    gnutar="gtar"
-  else
-    gnutar="tar"
-  fi
-  stream=$($gnutar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
-
-  OIFS=$IFS
-  IFS='|'
-  stream="${stream:1}"
-  current=""
-  for package in $stream
-  do
-    IFS='
-'
-    count=0
-    for line in $package; do
-      if [ $count -eq 0 ]; then
-        current="$current $line"
-      else
-        continue
-      fi
-
-      ((count++))
-    done
-
-    IFS='|'
-  done
-  IFS=$OIFS
-
-  cd $repo_directory
-  reg="^($repo_name.).+"
-  for filename in *; do
-    if [[ $filename =~ $reg || $filename = 'sync' || $filename = 'local'  || $filename = 'resources' ]]; then
-      continue
-    fi
-    for package in $current; do
-      if [ $filename = $package ]; then
-        continue 2
-      fi
-    done
-    echo "Removing $filename..."
-    rm $filename
-  done
-}
-
-run_directory_path=$(pwd)
-
-# Main process
-
-if [ "$1" = "" ]; then
-  init
-  clearRepo
-else
-  echo "Please call this script without any arguments."
-  exit 1
-fi
-
-exit 0

update-repo.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # The purpose of this script is to automate the maintenance of a custom pacman (Arch Linux) repository, whose goal is to
 # provide package-ready AUR softs.
@@ -9,7 +9,6 @@
 # Make sure the following packages are installed before running this script:
 # package-query aurutils git
 
-
 # Functions
 
 ##################################################################
@@ -83,11 +82,6 @@ init() {
   fi
 
   source ../variables.conf
-  repo_db_file=$repo_name.db.tar
-  repo_db_filesfile=$repo_name.files.tar
-  awk_rules_file='../parser.awk'
-  eval "$(ssh-agent)"
-  ssh-add $ssh_key
 
   # TODO Check if required packages are installed
 
@@ -97,8 +91,7 @@ init() {
 checkUpdates() {
   echo "Checking for updates:"
   upgraded_packages=""
-  scp $ssh_options $remote_repository/$repo_db_file $remote_repository/$repo_db_filesfile $run_directory_path/
+  stream=$(tar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
-  stream=$(tar xOf $run_directory_path/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
 
   OIFS=$IFS
   IFS='|'
@@ -126,12 +119,6 @@ checkUpdates() {
       continue 1
     fi
 
-    reg=".+-(debug)"
-    if [[ $name =~ $reg ]]; then
-      echo -e "\nSkipping $name ($version, default exception)."
-      continue 1
-    fi
-
     if echo $exlist | grep -w $name > /dev/null; then
       echo -e "\nSkipping $name ($version, temporary exception)."
       continue 1
@@ -144,7 +131,6 @@ checkUpdates() {
     result=$?
     if [[ $result -eq 1 ]]; then
       echo "New version for $name: $version -> $aur_version"
-      IFS=$OIFS
       createPackage $name
       addPackageToRepo $name $aur_version
       upgraded_packages="$upgraded_packages $name"
@@ -171,22 +157,27 @@ createPackage() { # Create package $1
   aur fetch $1 # TODO This could be replaced by a git clone or even a wget+tar thanks to package-query
   echo "Making $1 package..."
   cd $1
+  # if linux-lts{414,49} then download kernel source from local repo (saves ~100MB each time)
+  # WARNING this onlys works because we already have said sources AND because PKBUILD follow this particular line
+  # (not the same in 419 nor 54)
+  reg="(linux-lts).+"
+  if [[ $1 =~ $reg ]]; then
+    sed -i -e "s/\"https:\/\/www.kernel.org\/pub\/linux\/kernel\/v4.x\/\${_srcname}.tar.xz\"/\"https:\/\/archlinux.kto.black\/resources\/\${_srcname}.tar.xz\"/g" ./PKGBUILD
+  fi
   makepkg -s --noconfirm --noprogressbar
   cd $run_directory_path
 }
 
-# TODO: try manually: can we update db file before sending to remote repo?
 addPackageToRepo() { # Add package $1
   echo "Adding $1 to repository..."
-  scp $ssh_options $run_directory_path/$1/$1-$2*.pkg.tar.* $remote_repository/
+  cp $run_directory_path/$1/$1-$2*.pkg.tar.xz $repo_directory
-  repo-add $repo_db_file $run_directory_path/$1/$1-$2*.pkg.tar.*
+  repo-add $repo_directory/$repo_db_file $repo_directory/$1-$2*.pkg.tar.xz
   reg="(linux-lts).+"
   if [[ $name =~ $reg ]]; then
     echo "Adding $1-headers and $1-docs to repository..."
-    repo-add $repo_db_file $run_directory_path/$1/$1-headers-$2*.pkg.tar.* $run_directory_path/$1/$1-docs-$2*.pkg.tar.*
+    cp $run_directory_path/$1/$1-headers-$2*.pkg.tar.xz $run_directory_path/$1/$1-docs-$2*.pkg.tar.xz $repo_directory
-    scp $ssh_options $run_directory_path/$1/$1-headers-$2*.pkg.tar.* $run_directory_path/$1/$1-docs-$2*.pkg.tar.* $remote_repository/
+    repo-add $repo_directory/$repo_db_file $repo_directory/$1-headers-$2*.pkg.tar.xz $repo_directory/$1-docs-$2*.pkg.tar.xz
   fi
-  scp $ssh_options $run_directory_path/$repo_db_file $run_directory_path/$repo_db_file.old $run_directory_path/$repo_db_filesfile $run_directory_path/$repo_db_filesfile.old $remote_repository/
 }
 
 clean() {
@@ -195,8 +186,58 @@ clean() {
   echo "Done."
 }
 
+clearRepo() {
+  stream=$(tar xOf $repo_directory/$repo_db_file --wildcards */desc | awk -f $awk_rules_file)
+
+  OIFS=$IFS
+  IFS='|'
+  stream="${stream:1}"
+  current=""
+  for package in $stream
+  do
+    IFS='
+'
+    count=0
+    for line in $package; do
+      if [ $count -eq 0 ]; then
+        current="$current $line"
+      else
+        continue
+      fi
+
+      ((count++))
+    done
+
+    IFS='|'
+  done
+  IFS=$OIFS
+
+  cd $repo_directory
+  reg="^($repo_name.).+"
+  for filename in *; do
+    if [[ $filename =~ $reg || $filename = 'sync' || $filename = 'local'  || $filename = 'resources' ]]; then
+      continue
+    fi
+    for package in $current; do
+      if [ $filename = $package ]; then
+        continue 2
+      fi
+    done
+    echo "Removing $filename..."
+    rm $filename
+  done
+
+}
+
 run_directory_path=$(pwd)
 
+# Set parameters
+repo_directory='/mnt/archrepo'
+repo_name='kto'
+repo_db_file=$repo_name.db.tar
+awk_rules_file='../parser.awk'
+#repo_db_file='kto.db.tar' # You can set db name manually here if different
+
 # Main process
 
 if [ "$1" = "" ]; then
@@ -204,15 +245,18 @@ if [ "$1" = "" ]; then
   checkUpdates
 elif [ "$1" = "add" ]; then
   init
-  scp $ssh_options $remote_repository/$repo_db_file $remote_repository/$repo_db_filesfile $run_directory_path/
+  addPackage $2
-  for arg in "$@"; do
-    if [[ $arg = "add" ]]; then
-      continue
-    fi
-    addPackage $arg
-  done
 elif [ "$1" = "clean" ]; then
   clean
+elif [ "$1" = "clear" ]; then
+  read -r -p "Are you sure? [y/N] " response
+  response=${response,,}    # tolower
+  if [[ "$response" =~ ^(yes|y)$ ]]; then
+    clearRepo
+  else
+    echo "Cancelling... Perhaps did you mean 'clean', which empties the local build directory?"
+    exit 1
+  fi
 else
   exit 1
 fi

variables.conf.template

@@ -1,15 +1,5 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # Copy this file as variables.conf, with the correct values
 
-# For both update and clear scripts:
-repo_name=""
-
-# For update script:
-remote_repository=""
-ssh_key=""
-ssh_options="-i $ssh_key"
 exlist=""
-
-# For clear script:
-repo_directory=""