Bash Script : Sync Daily ISO

If you are using the daily ISO, please report issues here to keep confusion down in other forums. Questions or issues with Molecule also here

Moderator: Moderators

Bash Script : Sync Daily ISO

Postby sabayonino » Thu Oct 06, 2016 3:53

Hi guys

Here a simple bash script to update your Daily ISO (DAILY+DEV)
(using rsync)

an idea based on Wolfden's Script (Pyhton)

modified : 2016-10-13
Code: Select all
#!/bin/bash
clear
########################################################
# Where ISOs are stored
# Change with your favourite path

DEST="/home/nykay/iso"

# Check if destination folder exists.
if [[ ! -d "$DEST" ]] ; then
    echo -e $WHITE"Creating $LGREEN$DEST$WHITE destination directory..."$DEF
    mkdir -p "$DEST"
    sleep 3
    clear
fi
#########################################################

# Colorized the script   :)
# Foreground Colors
DEF="\e[39m"
RED="\e[31m"
GREEN="\e[32m"
YELLOW="\e[33m"
BLUE="\e[34m"
MAGEN="\e[35"
CYAN="\e[36m"
LGREY="\e[37m"
DGREY="\e[90m"
LRED="\e[91m"
LGREEN="\e[92m"
LYELLOW="\e[93m"
LBLUE="\e[94m"
LMAGEN="\e[95m"
LCYAN="\e[96m"
WHITE="\e[97m"
# End colorized the script :)
##########################################################

###################### Welcome Banner #######################
echo -e $LGREEN" ***** Welcome to Sabayon Daily Sync ISO ***** "
echo "           https://www.sabayon.org"
echo -e $DEF
####################### End Welcome Banner ###################
               
# Start path command ** not recognized inside "select" statement or various  loop statement **
# Check full path for curl and rsync
CURL=$(which curl 2>/dev/null)
RSYNC=$(which rsync 2>/dev/null)
# Check full path for grep and cut (needed for mirror selection)
CUT=$(which cut 2>/dev/null)
GREP=$(which grep 2>/dev/null)
CAT=$(which cat 2>/dev/null)
# Other
CLEAR=$(which clear 2>/dev/null)
SLEEP=$(which sleep 2>/dev/null)
RM=$(which rm 2>/dev/null)
# End path command

# Start URL definition
# Default RSYNC URL (uppercase for rsync)
DEFURL="rsync://mirror1.mirror.garr.it/sabayonlinux"
URL="$DEFURL"

# Default HTTP/FTP URL (lowercase for http/ftp)
defurl="http://mirror.it.sabayon.org/"
url="$defurl"

# HTTP/FTP mirrors in array
# Default is http://mirror.it.sabayon.org/
url1=("ftp://mirrors.coopvgg.com.ar/sabayon/" \
"http://mirrors.coopvgg.com.ar/sabayon/" \
"ftp://gd.tuwien.ac.at/linux/sabayonlinux/" \
"http://gd.tuwien.ac.at/linux/sabayonlinux/" \
"http://mirror.chmuri.net/sabayon/" \
"ftp://mirror.optusnet.com.au/sabayon/" \
"http://mirror.optusnet.com.au/sabayon/" \
"ftp://mirror.internode.on.net/pub/sabayon/" \
"http://mirror.internode.on.net/pub/sabayon/" \
"ftp://ftp.belnet.be/mirror/sabayonlinux" \
"http://ftp.belnet.be/sabayonlinux" \
"ftp://sabayon.c3sl.ufpr.br/sabayon" \
"http://sabayon.c3sl.ufpr.br/" \
"ftp://mirror.dkm.cz/sabayon/" \
"http://mirrors.cicku.me/sabayon/" \
"ftp://ftp.klid.dk/sabayonlinux/" \
"http://ftp.klid.dk/sabayonlinux/" \
"ftp://mirror.uta.edu.ec/sabayon" \
"http://mirror.uta.edu.ec/sabayon" \
"ftp://ftp.cc.uoc.gr/mirrors/linux/SabayonLinux/" \
"http://ftp.cc.uoc.gr/mirrors/linux/SabayonLinux/" \
"ftp://ftp.fsn.hu/pub/linux/distributions/sabayon/" \
"http://ftp.fsn.hu/pub/linux/distributions/sabayon/" \
"http://mirror.it.sabayon.org/" \
"ftp://mirror1.mirror.garr.it/sabayonlinux" \
"ftp://mirror2.mirror.garr.it/sabayonlinux" \
"ftp://mirror3.mirror.garr.it/sabayonlinux" \
"http://mirror1.mirror.garr.it/mirrors/sabayonlinux/" \
"http://mirror2.mirror.garr.it/mirrors/sabayonlinux/" \
"http://mirror3.mirror.garr.it/mirrors/sabayonlinux/" \
"ftp://ftp.kddilabs.jp/Linux/packages/sabayonlinux/" \
"http://ftp.kddilabs.jp/Linux/packages/sabayonlinux/" \
"http://ftp.tsukuba.wide.ad.jp/Linux/sabayon/" \
"ftp://ftp.riken.jp/Linux/sabayon/" \
"http://ftp.riken.jp/Linux/sabayon/" \
"ftp://ftp.nluug.nl/pub/os/Linux/distr/sabayonlinux" \
"ftp://ftp.surfnet.nl/pub/os/Linux/distr/sabayonlinux" \
"http://ftp.nluug.nl/os/Linux/distr/sabayonlinux" \
"http://mirror.auckland.ac.nz/sabayon/" \
"ftp://glua.ua.pt/sabayon/" \
"ftp://ftp.rnl.tecnico.ulisboa.pt/pub/sabayon" \
"http://ftp.rnl.tecnico.ulisboa.pt/pub/sabayon" \
"ftp://mirror.yandex.ru/sabayon/" \
"http://mirror.yandex.ru/sabayon/" \
"ftp://ftp.portlane.com/pub/os/linux/sabayon/" \
"http://ftp.portlane.com/pub/os/linux/sabayon/" \
"ftp://mirror.cs.vt.edu/pub/SabayonLinux/" \
"http://mirror.cs.vt.edu/pub/SabayonLinux/" \
"http://cross-lfs.sabayonlinux.org/" \
"http://mirror.clarkson.edu/sabayon/" \
"http://mirror.umd.edu/sabayonlinux/" \
"http://distro.ibiblio.org/sabayonlinux/" \
"http://mirror.freedif.org/Sabayon/" \
"ftp://sabayon.mirror.ac.za/" \
"http://sabayon.mirror.ac.za/" \
"Quit")

#Mirror remote path - daily -
PATH="iso/daily/"

##########################################
# Mirror selection
containsElement ()  {
local e
for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
    return 1
}

selectmirror()   {
# fill array with rsync mirrors
# mirrors+=( $($CURL -s https://www.sabayon.org/download | $GREP -o -E 'href="(rsync[^"#]+)"' | $CUT -d'"' -f2) )
# 'rsync://ftp.kddilabs.jp/sabayonlinux/')       # dead
# 'rsync://gd.tuwien.ac.at/linux/sabayonlinux/') # dead
# 'rsync://mirror.clarkson.edu/sabayonlinux/')   # dead
# 'rsync://mirror.umd.edu/sabayonlinux/')        # no daily
# 'rsync://mirror.uta.edu.ec/sabayon/')          # no daily
# 'rsync://sabayon.c3sl.ufpr.br/sabayon/'        # no daily

mirrors=('rsync://ftp.fsn.hu/linux/sabayon/' \
'rsync://ftp.nluug.nl/sabayonlinux/' \
'rsync://ftp.rnl.tecnico.ulisboa.pt/pub/sabayon/' \
'rsync://ftp.SURFnet.nl/sabayonlinux/' \
'rsync://ftp.tsukuba.wide.ad.jp/sabayon/' \
'rsync://mirror1.mirror.garr.it/sabayonlinux/' \
'rsync://mirrors.coopvgg.com.ar/sabayon/' \
'rsync://mirror.yandex.ru/sabayon/' \
'rsync://rsync.belnet.be/sabayon/' \
'rsync://sabayon.mirror.ac.za/sabayon' \
'Quit')

echo -e $CYAN

prompt=">> [Mirror] Option N°  > "
PS3="$prompt "
select opt in "${mirrors[@]}" ; do

    containsElement $opt "${mirrors[@]}"
    if [ $? -eq 0 ]; then
        if [ $opt == ${mirrors[-1]} ]; then
            break
        else
            URL="$opt"
# Get official timestamp for daily
            echo -e $LBLUE ""
            echo -e $LBLUE"Official DAILY url : $TIMESTAMPURL"
            echo -e $LBLUE"Official DAILY date: $TIMESTAMP"
# get mirror timestamp
            $RM ${DEST}/MIRROR_RELEASE_DATE_DAILY 2> /dev/null
            $RSYNC -qa $opt/iso/daily/RELEASE_DATE_DAILY ${DEST}/MIRROR_RELEASE_DATE_DAILY
            echo -e $LYELLOW"Mirror DAILY date: $($CAT ${DEST}/MIRROR_RELEASE_DATE_DAILY)"
            echo -e $LYELLOW"Mirror DAILY url : $opt"$DEF
                if [[ "$($CAT $DEST/MIRROR_RELEASE_DATE_DAILY)" != "$TIMESTAMP" ]] ; then
                    echo -n -e $LRED"Warning !!! Selected mirror is older than Offical Mirror.Can't sync.\nBack to the Menu list in "$DEF
                    for i in {5..1} ; do echo -n -e "$i " ; $SLEEP 1 ; done
                    echo -e $DEF
                    $CLEAR
                    URL="$DEFURL"
                    REPLY=
                fi
            break
        fi
    else
        echo "unrecognized selection, try again!"
    fi
done
echo -e $DEF
PS3=">> Option N° > "
}

echo -e $DEF"Current RSYNC mirror for ISO is: $LMAGEN $URL"
echo -e $DEF"Current FTP/HTTP mirror for TAR.GZ is: $LMAGEN $url"
echo -e $DEF
### end of mirror selection

# Get timestamp for daily
TIMESTAMPURL="$defurl"
TIMESTAMP=$($CURL  -s $TIMESTAMPURL/iso/daily/RELEASE_DATE_DAILY)
echo -e $LBLUE"Release date DAILY "
echo $TIMESTAMP
echo -e $WHITE
# ISOs name
ISO=("Change HTTP/FTP Mirror for TAR.GZ" \
"Change RSYNC mirror for ISO" \
"Sabayon_Linux_DAILY_amd64_ForensicsXfce" \
"Sabayon_Linux_DAILY_amd64_GNOME-dev" \
"Sabayon_Linux_DAILY_amd64_KDE-dev" \
"Sabayon_Linux_DAILY_amd64_LXQt-dev" \
"Sabayon_Linux_DAILY_amd64_MATE-dev" \
"Sabayon_Linux_DAILY_amd64_Minimal-dev" \
"Sabayon_Linux_DAILY_amd64_Server-dev" \
"Sabayon_Linux_DAILY_amd64_SpinBase-dev" \
"Sabayon_Linux_DAILY_amd64_Xfce-dev" \
"Sabayon_Linux_DAILY_amd64_GNOME" \
"Sabayon_Linux_DAILY_amd64_KDE" \
"Sabayon_Linux_DAILY_amd64_LXQt" \
"Sabayon_Linux_DAILY_amd64_MATE" \
"Sabayon_Linux_DAILY_amd64_Minimal" \
"Sabayon_Linux_DAILY_amd64_Server" \
"Sabayon_Linux_DAILY_amd64_SpinBase" \
"Sabayon_Linux_DAILY_amd64_Xfce" \
"Sabayon_Linux_DAILY_amd64_tarball.tar.gz" \
"Other - Sync all Sabayon-dev ISO" \
"Other - Sync all Sabayon-daily ISO" \
"Other - Sync all Sabayon ISO (dev-daily-tar.gz)" \
"Quit")

GETISO () {
                echo -e $LYELLOW"Downloading $WHITE $opt.iso" $LGREEN
                echo -e $LYELLOW"Mirror $WHITE $URL" $DEF
                # WARNING!!!! rsync://mirrors.coopvgg.com.ar/sabayon don't want -z rsync option
                OPTS="-avthhP --exclude=*"
                ${RSYNC} --include=$opt.{iso,iso.md5,iso.pkglist}  $OPTS $URL/$PATH/ $DEST/
                echo -e $DEF
                echo
                }

URLTAR ()   {
                        PS3=">> Select mirror > "
                        $CLEAR
                        url=""
                        if [[ "$opt" == "Change HTTP/FTP Mirror for TAR.GZ" ]] ; then
                            echo -e $DGREY
                            select url in "${url1[@]}" ; do
                            containsElement "$url" "${url1[@]}"
                            if [ $? -ne 0 ] ; then
                                echo "FTP/HTTP URL malformed"
                            else
                                break
                            fi
                            echo -e $DEF
                            done
                        fi
                        if [[ -z "$url"  || "$url" == "Quit" ]] ; then url="$defurl" ; fi
                        PS3=">> Selection > "
                        }
GETTAR ()   {
                        opt="Sabayon_Linux_DAILY_amd64_tarball.tar.gz"
                        echo -e $LYELLOW"Downloading $WHITE $opt " $LGREEN
                        echo -e $LYELLOW"Mirror $WHITE $url" $DEF
                        $CURL -o $DEST/$opt $url/$PATH/$opt
                        $CURL -o $DEST/$opt.md5 $url/$PATH/$opt.md5
                        $CURL -o $DEST/$opt.pkglist $url/$PATH/$opt.pkglist
                        echo -e $DEF
                        echo
                        $CLEAR ; REPLY=
                    }

echo -e $LGREY
PS3=">> Option N° > "
select opt in "${ISO[@]}" ; do
    case $opt in
        "${ISO[0]}") URLTAR  && REPLY= ;;
        "${ISO[1]}") selectmirror && REPLY= ;;
        "${ISO[2]}") GETISO ; REPLY= ;;
        "${ISO[3]}") GETISO ; REPLY=  ;;
        "${ISO[4]}") GETISO ; REPLY=  ;;
        "${ISO[5]}") GETISO ; REPLY=  ;;
        "${ISO[6]}") GETISO ; REPLY=  ;;
        "${ISO[7]}") GETISO ; REPLY=  ;;
        "${ISO[8]}") GETISO ; REPLY=  ;;
        "${ISO[9]}") GETISO ; REPLY=  ;;
        "${ISO[10]}") GETISO ; REPLY=  ;;
        "${ISO[11]}") GETISO ; REPLY=  ;;
        "${ISO[12]}") GETISO ; REPLY=  ;;
        "${ISO[13]}") GETISO ; REPLY=  ;;
        "${ISO[14]}") GETISO ; REPLY=  ;;
        "${ISO[15]}") GETISO ; REPLY=  ;;
        "${ISO[16]}") GETISO ; REPLY=  ;;
        "${ISO[17]}") GETISO ; REPLY=  ;;
        "${ISO[18]}") GETISO ; REPLY=  ;;
        "${ISO[19]}") GETTAR ; REPLY=  ;;
        "${ISO[20]}") for opt in "${ISO[@]:3:8}" ; do GETISO ; done ; REPLY=  ;;
        "${ISO[21]}") for opt in "${ISO[@]:11:8}" ; do GETISO ; done ; REPLY=  ;;
        "${ISO[22]}") for opt in "${ISO[@]:2:17}" ; do GETISO ; done ; GETTAR ; REPLY=  ;;
        "${ISO[23]}") $CLEAR
                    echo "Thanks for using Sabayon Sync ISO"
                    $SLEEP 5
                    break ;;
    *) echo -e $LRED "invalid option" ;;
    esac
    echo -e $DEF
done
User avatar
sabayonino
Sagely Hen
 
Posts: 3000
Joined: Sun Sep 21, 2008 1:12
Location: Italy

Re: Bash Script : Sync Daily ISO

Postby nomorelogic » Sat Oct 08, 2016 8:39

sabayonino wrote:feel free to improve it :roll:


Hi,
I realized a little improvement to your script, now you can select a mirror.
The list is currently taken from https://www.sabayon.org/download, maybe can be useful :)

regards
nomorelogic

Code: Select all
#!/bin/bash
clear

# Foreground Colors
DEF="\e[39m"
RED="\e[31m"
GREEN="\e[32m"
YELLOW="\e[33m"
BLUE="\e[34m"
MAGEN="\e[35"
CYAN="\e[36m"
LGREY="\e[37m"
DGREY="\e[90m"
LRED="\e[91m"
LGREEN="\e[92m"
LYELLOW="\e[93m"
LBLUE="\e[94m"
LMAGEN="\e[95m"
LCYAN="\e[96m"
WHITE="\e[97m"

echo -e $LGREEN" ***** Welcome to Sabayon Daily Sync ISO ***** "
echo "           https://www.sabayon.org"
echo -e $DEF

# Check full path for curl and rsync
CURL=$(whereis curl 2>/dev/null | awk '{print $2}')
RSYNC=$(whereis rsync 2>/dev/null | awk '{print $2}')

# Check full path for cur, grep and curl (needed for mirror selection)
CUT=$(whereis cut 2>/dev/null | awk '{print $2}')
GREP=$(whereis -b grep 2>/dev/null|tr ' ' '\n'|grep "grep$")
CURL=$(whereis -b curl 2>/dev/null | awk '{print $2}')

# Rsync URL
# see https://www.sabayon.org/download   rsync server for your timezone
URL="rsync://mirror1.mirror.garr.it/sabayonlinux"


##########################################

# Mirror selection


containsElement () {
  local e
  for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
  return 1
}


selectmirror()
{
   # fill array with rsync mirrors
  mirrors=()
  mirrors+=( $($CURL -s https://www.sabayon.org/download | $GREP -o -E 'href="(rsync[^"#]+)"' | $CUT -d'"' -f2) )
  mirrors+=('Quit')

   echo "Select a mirror"
   prompt="Select:"
   PS3="$prompt "
   select opt in "${mirrors[@]}" ; do

    containsElement $opt "${mirrors[@]}"

    if [ $? -eq 0 ]; then
       echo "Selected rsync mirror is: $opt"
          URL="$opt"
          break
      else
          echo "unrecognized selection, try again!"
      fi

   done

}

echo Selected rsync mirror is: $URL

#### end of mirror selection


# Remote path
PATH="iso/daily/"

# Options for rsync
OPTS="--progress --partial -avr --exclude={community,distfiles,entropy,testing}"

##############################

# Where ISOs are stored
# Change with your favourite path

DEST="/media/dati/Virtual/iso/Sabayon/"

##############################

# Get timestamp for daily
# Set your favourite mirror
# https://www.sabayon.org/download
TIMESTAMP=$(${CURL}  http://na.mirror.garr.it/mirrors/sabayonlinux/iso/daily/RELEASE_DATE_DAILY > ${DEST}/RELEASE_DATE_DAILY 2>/dev/null)

echo -e $LBLUE"Release date DAILY "
echo | /bin/cat ${DEST}/RELEASE_DATE_DAILY

# ISOs name
OPT=("Change mirror" "Sabayon_Linux_DAILY_amd64_ForensicsXfce" \
"Sabayon_Linux_DAILY_amd64_GNOME" \
"Sabayon_Linux_DAILY_amd64_GNOME-dev" \
"Sabayon_Linux_DAILY_amd64_KDE" \
"Sabayon_Linux_DAILY_amd64_KDE-dev" \
"Sabayon_Linux_DAILY_amd64_LXQt" \
"Sabayon_Linux_DAILY_amd64_LXQt-dev" \
"Sabayon_Linux_DAILY_amd64_MATE" \
"Sabayon_Linux_DAILY_amd64_MATE-dev" \
"Sabayon_Linux_DAILY_amd64_Minimal" \
"Sabayon_Linux_DAILY_amd64_Minimal-dev" \
"Sabayon_Linux_DAILY_amd64_Server" \
"Sabayon_Linux_DAILY_amd64_Server-dev" \
"Sabayon_Linux_DAILY_amd64_SpinBase" \
"Sabayon_Linux_DAILY_amd64_SpinBase-dev" \
"Sabayon_Linux_DAILY_amd64_Xfce" \
"Sabayon_Linux_DAILY_amd64_Xfce-dev" \
"Sabayon_Linux_DAILY_amd64_tarball.tar.gz" \
"Quit")

MESG () {
            echo -e $LYELLOW "Downloading $WHITE $opt.iso" $LGREEN
            ${RSYNC} $OPTS $URL/$PATH/$opt.{iso,iso.md5,iso.pkglist} ${DEST}/
            echo -e $DEF
            echo
}

MESGTAR () {
            echo -e $LYELLOW "Downloading $WHITE $opt " $LGREEN
            ${RSYNC} $OPTS $URL/$PATH/$opt{.md5,.pkglist} ${DEST}/
            echo -e $DEF
            echo
}


echo -e $LGREY
PS3='Please enter your choice : '
select opt in "${OPT[@]}"
do
case $opt in
        "Change mirror") selectmirror && REPLY= ;;
        "Sabayon_Linux_DAILY_amd64_ForensicsXfce") MESG ;;
        "Sabayon_Linux_DAILY_amd64_GNOME") MESG ;;
        "Sabayon_Linux_DAILY_amd64_GNOME-dev") MESG  ;;
        "Sabayon_Linux_DAILY_amd64_KDE") MESG ;;
        "Sabayon_Linux_DAILY_amd64_KDE-dev") MESG ;;
        "Sabayon_Linux_DAILY_amd64_LXQt") MESG ;;
        "Sabayon_Linux_DAILY_amd64_LXQt-dev") MESG  ;;
        "Sabayon_Linux_DAILY_amd64_MATE") MESG ;;
        "Sabayon_Linux_DAILY_amd64_MATE-dev") MESG ;;
        "Sabayon_Linux_DAILY_amd64_Minimal") MESG ;;
        "Sabayon_Linux_DAILY_amd64_Minimal-dev") MESG ;;
        "Sabayon_Linux_DAILY_amd64_Server") MESG ;;
        "Sabayon_Linux_DAILY_amd64_Server-dev") MESG ;;
        "Sabayon_Linux_DAILY_amd64_SpinBase") MESG ;;
        "Sabayon_Linux_DAILY_amd64_SpinBase-dev") MESG ;;
        "Sabayon_Linux_DAILY_amd64_Xfce") MESG  ;;
        "Sabayon_Linux_DAILY_amd64_Xfce-dev") MESG  ;;
        "Sabayon_Linux_DAILY_amd64_tarball.tar.gz") MESGTAR  ;;
        "Quit") /usr/bin/clear
                    echo "Thanks for using Sabayon Sync ISO"
                    /usr/bin/sleep 5
                    break ;;
        *) echo -e $LRED "invalid option" ;;
    esac
    echo -e $DEF
done
nomorelogic
Simple Hen
 
Posts: 68
Joined: Sat Jan 14, 2012 13:40

Re: Bash Script : Sync Daily ISO

Postby sabayonino » Sat Oct 08, 2016 11:04

thanks nomorelogic


edit : "which" command to get exutables (curl,cut,rsync...) instead "whereis"
not mandatory.




Code: Select all
@ERROR: Unknown module 'sabayonlinux'
rsync error: error starting client-server protocol (code 5) at main.c(1648) [Receiver=3.1.2]

not of all...

some mirrors could be not updated
Code: Select all
rsync -av rsync://mirror.clarkson.edu/sabayonlinux ./mirror

@ERROR: Unknown module 'sabayonlinux'
rsync error: error starting client-server protocol (code 5) at main.c(1648) [Receiver=3.1.2]


"module" passed by rsync is wrong

some mirror has "sabayon" as module
others not
so $PATH colud be "iso/daily" or "sabayon/iso/daily" (depend on the rsync mirror)


exsample

rsync://sabayon.mirror.ac.za mirror (get by the list) must be add "sabayon" module (a mirrror could have more module available , ubuntu,fedora....other and so on)
Code: Select all
$ rsync -avn rsync://sabayon.mirror.ac.za/sabayon ./
receiving incremental file list
./
community/
community/AVAILABLE_REPOSITORIES
community/metadata.json
community/community-binhost/
community/community-binhost/Packages


Code: Select all
rsync -avn rsync://sabayon.mirror.ac.za/ubuntu ./
receiving incremental file list
./
ubuntu-cdimage/
ubuntu-cdimage/.htaccess
ubuntu-cdimage/.manifest-daily
ubuntu-cdimage/favicon.ico


rsync://rsync.belnet.be (rsync://rsync.belnet.be/sabayon/ on the list , sabayon is called "module")
Code: Select all
$ rsync -avn rsync://rsync.belnet.be ./
centos          CentOS mirror (approx 94 GB)
cpan            CPAN mirror (approx 13 GB)
debian          Debian (approx 610 GB)
deepin          Linux Deepin packages (approx 47 GB)
deepin-cd       Linux Deepin ISO images (approx 52 GB)
freebsd         FreeBSD mirror (approx 875 GB)
gentoo          Gentoo mirror (approx 217 GB)
kali            The Kali Archive (approx 442 GB)
kali-security   The Kali security archive (approx 443 GB)
kali-images     The Kali ISO images (approx 39 GB)
lubuntu         Xubuntu mirror (approx 35 GB)
mageia          Mageia mirror (approx 167 GB)
mptcp           Multipath TCP project mirror (approx 2 GB)
openbsd         OpenBSD mirror (approx 1.2 TB)
openelec.tv     OpenELEC mirror (approx 3.5 GB)
sabayon         Sabayon mirror (approx 241 GB)
slackware       Slackware mirror (approx 159 GB)
ubuntu          Ubuntu mirror (approx 275 GB)
videolan        Videolan mirror (approx 25 GB)
xubuntu         Xubuntu mirror (approx 35 GB)
User avatar
sabayonino
Sagely Hen
 
Posts: 3000
Joined: Sun Sep 21, 2008 1:12
Location: Italy

Re: Bash Script : Sync Daily ISO

Postby sabayonino » Sat Oct 08, 2016 12:35

selecting "Quit" in rsync selection

URL is set to "Quit"


Code: Select all
if [ $? -eq 0 ]; then
               echo "Selected rsync mirror is: $opt"
                    if [[ "$opt" != "Quit" ]] ; then
                        URL="$opt"
                    fi
                  break
              else
                  echo "unrecognized selection, try again!"
              fi

           done
User avatar
sabayonino
Sagely Hen
 
Posts: 3000
Joined: Sun Sep 21, 2008 1:12
Location: Italy

Re: Bash Script : Sync Daily ISO

Postby nomorelogic » Sat Oct 08, 2016 12:46

with a bit of time I can debug this and I can try to patch the script
but ...
getting the list of modules of clarkson mirror I can't see sabayon

Code: Select all
$ rsync rsync://mirror.clarkson.edu

parrot            Parrot
gentoo            Gentoo
gentoo-distfiles   Gentoo
gentoo-portage    Gentoo Portage
centos            CentOS
gnu               GNU Software
tdf               The Document Foundation
dag               DAG/RPMForge/RepoForge
repoforge         DAG/RPMForge/RepoForge
rpmforge          DAG/RPMForge/RepoForge


it looks like sabayon module was removed?
nomorelogic
Simple Hen
 
Posts: 68
Joined: Sat Jan 14, 2012 13:40

Re: Bash Script : Sync Daily ISO

Postby sabayonino » Sat Oct 08, 2016 13:05

nomorelogic wrote:with a bit of time I can debug this and I can try to patch the script
but ...
getting the list of modules of clarkson mirror I can't see sabayon

Code: Select all
$ rsync rsync://mirror.clarkson.edu

parrot            Parrot
gentoo            Gentoo
gentoo-distfiles   Gentoo
gentoo-portage    Gentoo Portage
centos            CentOS
gnu               GNU Software
tdf               The Document Foundation
dag               DAG/RPMForge/RepoForge
repoforge         DAG/RPMForge/RepoForge
rpmforge          DAG/RPMForge/RepoForge


it looks like sabayon module was removed?


right , but some rsync-mirror (I can't remember) isn't updated or point to wrong module .
other mirror poitn directly to sabayon so $PATH is "iso/daily"
User avatar
sabayonino
Sagely Hen
 
Posts: 3000
Joined: Sun Sep 21, 2008 1:12
Location: Italy

Re: Bash Script : Sync Daily ISO

Postby sabayonino » Sat Oct 08, 2016 13:12

checkikng...
(-n option means DRYRUN)

clarkson is dead for sabayon
Code: Select all
rsync -avn rsync://mirror.clarkson.edu/ .

parrot          Parrot
gentoo          Gentoo
gentoo-distfiles        Gentoo
gentoo-portage  Gentoo Portage
centos          CentOS
gnu             GNU Software
tdf             The Document Foundation
dag             DAG/RPMForge/RepoForge
repoforge       DAG/RPMForge/RepoForge
rpmforge        DAG/RPMForge/RepoForge



rsync://mirror.uta.edu.ec/sabayon (has no daily) [Skip this for or purposes)
Code: Select all
receiving incremental file list
rsync: opendir "/.~tmp~" (in sabayon) failed: Permission denied (13)
./
mirror.umd.edu.size.txt
mirror.umd.edu.timestamp.txt
.~tmp~/
iso/
iso/TIMESTAMP
iso/main.html
iso/style.css
iso/monthly/
iso/monthly/LATEST_IS




rsync://gd.tuwien.ac.at (this mirror is dead for sabayon)
Code: Select all
rsync -avn rsync://gd.tuwien.ac.at .
rsync: failed to connect to gd.tuwien.ac.at (192.35.244.50): Connection refused (111)
rsync error: error in socket IO (code 10) at clientserver.c(125) [Receiver=3.1.2]



rsync://ftp.kddilabs.jp (has no sabayon)
Code: Select all
rsync -avn rsync://ftp.kddilabs.jp .
MOTD:
MOTD:  Welcome to KDDI Research, Inc.
MOTD:
MOTD:  This ftp/http/rsync server is an unsupported service of KDDI
MOTD:  Research, Inc. Use entirely at your own risk - no warranty
MOTD:  is expressed or implied. Complaints and questions should be
MOTD:  sent to admin -at- ftp.ne.jp.
MOTD:
MOTD:        ftp://ftp.kddilabs.jp       (ftp://ftp.ne.jp/)
MOTD:        http://ftp.kddilabs.jp      (http://ftp.ne.jp/)
MOTD:        rsync://ftp.kddilabs.jp     (rsync://ftp.ne.jp)
MOTD:
MOTD:   * You may not access some directories in a module by
MOTD:     the security. Please try ftparea-all module with
MOTD:     the specified directory in this case.
MOTD:      [EXAMPLE] The following is the same.
MOTD:         rsync://ftp.kddilabs.jp/fedora/
MOTD:         rsync://ftp.kddilabs.jp/ftparea-all/Linux/packages/fedora/
MOTD:
MOTD:   * The connection for both IPv4 and IPv6 is available.
MOTD:   * This server equipments and connections for the Internet
MOTD:     are supported by KDDI Research, Inc.
MOTD:
MOTD:

ftparea-all     *ALL* ftp.kddilabs.jp area. DO NOT USE THIS MODULE FOR ALL ARCHIVES SYNC. Please specify each directory for sync.
kernel-org      Kernel.ORG
linux-kernel    Linux Kernel
redhat          Red Hat linux
fedora          Fedora Project
debian          debian linux
suse            SuSE linux
opensuse        openSUSE Linux
turbolinux      Turbolinux archive
mandriva        Mandriva Linux (formely Mandrake)
plamo           Plamo Linux
centos          CentOS
ubuntu          ubuntu
Vine            Vine Linux
manjaro         manjaro linux
elrepo          ELRepo: RPM repository for Enterprise Linux
slackware       Slackware Linux Project
kali            Kali Linux (main)
kali-images     Kali Linux (images)
openbsd         OpenBSD
freebsd         FreeBSD
apache          Apache Software Foundation (incl. httpd)
X11             X Window (X10, X11)
kde             K desktop environment
gnome           GNOME
mysql           MySQL is the world's most popular open source database
tdf             TDF project
ctan            common tex archive network
tex             TeX related archives
cpan            common perl archive network


rsync://mirror.clarkson.edu/ (has no sabayon)
Code: Select all
rsync -avn rsync://mirror.clarkson.edu/ .

parrot          Parrot
gentoo          Gentoo
gentoo-distfiles        Gentoo
gentoo-portage  Gentoo Portage
centos          CentOS
gnu             GNU Software
tdf             The Document Foundation
dag             DAG/RPMForge/RepoForge
repoforge       DAG/RPMForge/RepoForge
rpmforge        DAG/RPMForge/RepoForge


rsync://mirror.umd.edu/sabayonlinux (has no daily path) (Skip this for our purposes)
Code: Select all
receiving incremental file list
./
mirror.umd.edu.size.txt
mirror.umd.edu.timestamp.txt
iso/
iso/TIMESTAMP
iso/main.html
iso/style.css
iso/monthly/
iso/monthly/LATEST_IS


rsync://sabayon.mirror.ac.za (need PATH="sabayon/iso/daily")
or update downloadpage adding "/sabayon" to rsync://sabayon.mirror.ac.za



going to file a bug in ML to edit download page :mrgreen:
User avatar
sabayonino
Sagely Hen
 
Posts: 3000
Joined: Sun Sep 21, 2008 1:12
Location: Italy

Re: Bash Script : Sync Daily ISO

Postby sabayonino » Sun Oct 09, 2016 0:25

update

Sabayon tarball can't be updated over rsync
Just adding http/ftp URLs in array (I hope all mirrors work !! :mrgreen: )
and split/modified MSGTAR function
User avatar
sabayonino
Sagely Hen
 
Posts: 3000
Joined: Sun Sep 21, 2008 1:12
Location: Italy

Re: Bash Script : Sync Daily ISO

Postby nomorelogic » Sun Oct 09, 2016 12:19

Hi sabayonino
I sent you a MP but there's something strange: I can't see the MP in "sended message"...
can you see it?
thanks
nomorelogic
Simple Hen
 
Posts: 68
Joined: Sat Jan 14, 2012 13:40

Re: Bash Script : Sync Daily ISO

Postby nomorelogic » Tue Oct 11, 2016 7:37

Hi, I discovered mirror rsync://mirrors.coopvgg.com.ar/sabayon/ don't accept -z option with rsync...
Even worse, daily mirror is not updated!

I realize a little change to show mirror timestamps *before* downloading

Example:
Official DAILY url : http://na.mirror.garr.it/mirrors/sabayonlinux
Official DAILY date: Mon Oct 10 09:22:46 UTC 2016
Mirror DAILY date: Wed Feb 25 05:49:43 UTC 2015
Mirror DAILY url : rsync://mirrors.coopvgg.com.ar/sabayon/

Code: Select all
#!/bin/bash
    clear

    ##############################

        # Where ISOs are stored
        # Change with your favourite path

    # DEST="/path/your/iso/"
    DEST="/media/dati/Virtual/iso/Sabayon"

    ##############################

        # Foreground Colors
        DEF="\e[39m"
        RED="\e[31m"
        GREEN="\e[32m"
        YELLOW="\e[33m"
        BLUE="\e[34m"
        MAGEN="\e[35"
        CYAN="\e[36m"
        LGREY="\e[37m"
        DGREY="\e[90m"
        LRED="\e[91m"
        LGREEN="\e[92m"
        LYELLOW="\e[93m"
        LBLUE="\e[94m"
        LMAGEN="\e[95m"
        LCYAN="\e[96m"
        WHITE="\e[97m"

        echo -e $LGREEN" ***** Welcome to Sabayon Daily Sync ISO ***** "
        echo "           https://www.sabayon.org"
        echo -e $DEF

        # Check full path for curl and rsync
        CURL=$(which curl 2>/dev/null)
        RSYNC=$(which rsync 2>/dev/null)

        # Check full path for grep and cut (needed for mirror selection)
        CUT=$(which cut 2>/dev/null)
        GREP=$(which grep 2>/dev/null)
        CAT=$(which cat 2>/dev/null)

        # Other
        CLEAR=$(which clear 2>/dev/null)
        SLEEP=$(which sleep 2>/dev/null)
        RM=$(which rm 2>/dev/null)

        # Default Rsync URL
        URL="rsync://mirror1.mirror.garr.it/sabayonlinux"
        # Default Http/Ftp URL
        url="http://mirror.it.sabayon.org/"

        # HTTP/FTP mirrors in array
        # Default is http://mirror.it.sabayon.org/
        URL1=("ftp://mirrors.coopvgg.com.ar/sabayon/" \
        "http://mirrors.coopvgg.com.ar/sabayon/" \
        "ftp://gd.tuwien.ac.at/linux/sabayonlinux/" \
        "http://gd.tuwien.ac.at/linux/sabayonlinux/" \
         "http://mirror.chmuri.net/sabayon/" \
        "ftp://mirror.optusnet.com.au/sabayon/" \
        "http://mirror.optusnet.com.au/sabayon/" \
        "ftp://mirror.internode.on.net/pub/sabayon/" \
        "http://mirror.internode.on.net/pub/sabayon/" \
        "ftp://ftp.belnet.be/mirror/sabayonlinux" \
        "http://ftp.belnet.be/sabayonlinux" \
        "ftp://sabayon.c3sl.ufpr.br/sabayon" \
        "http://sabayon.c3sl.ufpr.br/" \
        "ftp://mirror.dkm.cz/sabayon/"\
        "http://mirrors.cicku.me/sabayon/" \
        "ftp://ftp.klid.dk/sabayonlinux/" \
        "http://ftp.klid.dk/sabayonlinux/" \
        "ftp://mirror.uta.edu.ec/sabayon" \
        "http://mirror.uta.edu.ec/sabayon" \
        "ftp://ftp.cc.uoc.gr/mirrors/linux/SabayonLinux/" \
        "http://ftp.cc.uoc.gr/mirrors/linux/SabayonLinux/" \
        "ftp://ftp.fsn.hu/pub/linux/distributions/sabayon/" \
        "http://ftp.fsn.hu/pub/linux/distributions/sabayon/" \
        "http://mirror.it.sabayon.org/" \
        "ftp://mirror1.mirror.garr.it/sabayonlinux" \
        "ftp://mirror2.mirror.garr.it/sabayonlinux" \
        "ftp://mirror3.mirror.garr.it/sabayonlinux" \
        "http://mirror1.mirror.garr.it/mirrors/sabayonlinux/" \
        "http://mirror2.mirror.garr.it/mirrors/sabayonlinux/" \
        "http://mirror3.mirror.garr.it/mirrors/sabayonlinux/" \
        "ftp://ftp.kddilabs.jp/Linux/packages/sabayonlinux/" \
        "http://ftp.kddilabs.jp/Linux/packages/sabayonlinux/" \
        "http://ftp.tsukuba.wide.ad.jp/Linux/sabayon/" \
        "ftp://ftp.riken.jp/Linux/sabayon/" \
        "http://ftp.riken.jp/Linux/sabayon/" \
        "ftp://ftp.nluug.nl/pub/os/Linux/distr/sabayonlinux" \
        "ftp://ftp.surfnet.nl/pub/os/Linux/distr/sabayonlinux" \
        "http://ftp.nluug.nl/os/Linux/distr/sabayonlinux" \
        "http://mirror.auckland.ac.nz/sabayon/" \
        "ftp://glua.ua.pt/sabayon/" \
        "ftp://ftp.rnl.tecnico.ulisboa.pt/pub/sabayon" \
        "http://ftp.rnl.tecnico.ulisboa.pt/pub/sabayon" \
        "ftp://mirror.yandex.ru/sabayon/" \
        "http://mirror.yandex.ru/sabayon/" \
        "ftp://ftp.portlane.com/pub/os/linux/sabayon/" \
        "http://ftp.portlane.com/pub/os/linux/sabayon/" \
        "ftp://mirror.cs.vt.edu/pub/SabayonLinux/" \
        "http://mirror.cs.vt.edu/pub/SabayonLinux/" \
        "http://cross-lfs.sabayonlinux.org/" \
        "http://mirror.clarkson.edu/sabayon/" \
        "http://mirror.umd.edu/sabayonlinux/" \
        "http://distro.ibiblio.org/sabayonlinux/" \
        "http://mirror.freedif.org/Sabayon/" \
        "ftp://sabayon.mirror.ac.za/" \
        "http://sabayon.mirror.ac.za/"\
        "Quit")

        #Remote path
        PATH="iso/daily/"

        ##########################################

        # Mirror selection

        containsElement () {
          local e
          for e in "${@:2}"; do [[ "$e" == "$1" ]] && return 0; done
          return 1
        }

        selectmirror()
        {
          # fill array with rsync mirrors
          # mirrors+=( $($CURL -s https://www.sabayon.org/download | $GREP -o -E 'href="(rsync[^"#]+)"' | $CUT -d'"' -f2) )
          # 'rsync://ftp.kddilabs.jp/sabayonlinux/')       # dead
          # 'rsync://gd.tuwien.ac.at/linux/sabayonlinux/') # dead
          # 'rsync://mirror.clarkson.edu/sabayonlinux/')   # dead
          # 'rsync://mirror.umd.edu/sabayonlinux/')        # no daily
          # 'rsync://mirror.uta.edu.ec/sabayon/')          # no daily
          # 'rsync://sabayon.c3sl.ufpr.br/sabayon/'        # no daily

          mirrors=('rsync://ftp.fsn.hu/linux/sabayon/' \
                   'rsync://ftp.nluug.nl/sabayonlinux/' \
                   'rsync://ftp.rnl.tecnico.ulisboa.pt/pub/sabayon/' \
                   'rsync://ftp.SURFnet.nl/sabayonlinux/' \
                   'rsync://ftp.tsukuba.wide.ad.jp/sabayon/' \
                   'rsync://mirror1.mirror.garr.it/sabayonlinux/' \
                   'rsync://mirrors.coopvgg.com.ar/sabayon/' \
                   'rsync://mirror.yandex.ru/sabayon/' \
                   'rsync://rsync.belnet.be/sabayon/' \
                   'rsync://sabayon.mirror.ac.za/sabayon' \
                   'Quit')

           echo "Select a mirror"
           prompt="Select:"
           PS3="$prompt "
           select opt in "${mirrors[@]}" ; do

            containsElement $opt "${mirrors[@]}"

            if [ $? -eq 0 ]; then
               if [ $opt == ${mirrors[-1]} ]; then
                  break
               else
                     URL="$opt"

                  # Get official timestamp for daily
                  echo -e $LBLUE ""
                  echo -e $LBLUE"Official DAILY url : $TIMESTAMPURL"
                  echo -e $LBLUE"Official DAILY date: $TIMESTAMP"

                  # get mirror timestamp
                  $RM ${DEST}/MIRROR_RELEASE_DATE_DAILY 2> /dev/null
                  $RSYNC -qa $opt/iso/daily/RELEASE_DATE_DAILY ${DEST}/MIRROR_RELEASE_DATE_DAILY
                  echo -e $LYELLOW " Mirror DAILY date: $($CAT ${DEST}/MIRROR_RELEASE_DATE_DAILY)"
                  echo -e $LYELLOW " Mirror DAILY url : $opt"
                  $RM ${DEST}/MIRROR_RELEASE_DATE_DAILY 2> /dev/null

                  break
               fi
              else
                  echo "unrecognized selection, try again!"
              fi

           done

        }

        echo -e $DEF"Current RSYNC mirror for ISO is: $LMAGEN $URL"
        echo -e $DEF"Current FTP/HTTP mirror for TAR.BZ is: $LMAGEN $url"
        echo -e $DEF

        #### end of mirror selection

        # Options for rsync
        #OPTS="-avtzhhP --exclude={community,distfiles,entropy,testing}"

        # WARNING!!!! rsync://mirrors.coopvgg.com.ar/sabayon don't want -z rsync option
        OPTS="-avtzhhP --exclude={community,distfiles,entropy,testing}"


        # Get timestamp for daily
        TIMESTAMPURL="http://na.mirror.garr.it/mirrors/sabayonlinux"
        TIMESTAMP=$($CURL  -s $TIMESTAMPURL/iso/daily/RELEASE_DATE_DAILY)
        echo -e $LBLUE"Release date DAILY "
        echo $TIMESTAMP

        # ISOs name
        OPT=("Change HTTP/FTP Mirror for TAR.GZ" \
        "Change RSYNC mirror for ISO"\
        "Sabayon_Linux_DAILY_amd64_ForensicsXfce" \
        "Sabayon_Linux_DAILY_amd64_GNOME" \
        "Sabayon_Linux_DAILY_amd64_GNOME-dev" \
        "Sabayon_Linux_DAILY_amd64_KDE" \
        "Sabayon_Linux_DAILY_amd64_KDE-dev" \
        "Sabayon_Linux_DAILY_amd64_LXQt" \
        "Sabayon_Linux_DAILY_amd64_LXQt-dev" \
        "Sabayon_Linux_DAILY_amd64_MATE" \
        "Sabayon_Linux_DAILY_amd64_MATE-dev" \
        "Sabayon_Linux_DAILY_amd64_Minimal" \
        "Sabayon_Linux_DAILY_amd64_Minimal-dev" \
        "Sabayon_Linux_DAILY_amd64_Server" \
        "Sabayon_Linux_DAILY_amd64_Server-dev" \
        "Sabayon_Linux_DAILY_amd64_SpinBase" \
        "Sabayon_Linux_DAILY_amd64_SpinBase-dev" \
        "Sabayon_Linux_DAILY_amd64_Xfce" \
        "Sabayon_Linux_DAILY_amd64_Xfce-dev" \
        "Sabayon_Linux_DAILY_amd64_tarball.tar.gz" \
        "Quit")

        MESG ()     {
                    echo -e $LYELLOW"Downloading $WHITE $opt.iso" $LGREEN
                    echo -e $LYELLOW"Mirror $WHITE $URL" $DEF
                    ${RSYNC} $OPTS $URL/$PATH/$opt.{iso,iso.md5,iso.pkglist} $DEST/
                    echo -e $DEF
                    echo
        }

        URLTAR ()   {
                $CLEAR
               url=""
                if [[ "$OPT" == "Change HTTP/FTP Mirror for TAR.GZ" ]] ; then
                    select url in "${URL1[@]}"
                    do
                        containsElement "$url" "${URL1[@]}"
                            if [ $? -ne 0 ] ; then
                                echo "FTP/HTTP URL malformed"
                            else
                                break
                            fi
                    done
                fi
        }
        GETTAR ()   {
               opt="Sabayon_Linux_DAILY_amd64_tarball.tar.gz"
               if [[ -z "$url"  || "$url" == "Quit" ]] ; then url="http://mirror.it.sabayon.org/" ; fi
               echo -e $LYELLOW"Downloading $WHITE $opt " $LGREEN
               echo -e $LYELLOW"Mirror $WHITE $url" $DEF
               $CURL -o $DEST/$opt $url/$PATH/$opt
               $CURL -o $DEST/$opt.md5 $url/$PATH/$opt.md5
               $CURL -o $DEST/$opt.pkglist $url/$PATH/$opt.pkglist
               echo -e $DEF
               echo
         }

        echo -e $LGREY
        PS3='Please enter your choice : '
        select opt in "${OPT[@]}"
        do
        case $opt in
                "Change HTTP/FTP Mirror for TAR.GZ") URLTAR  ; GETTAR ;;
                "Change RSYNC mirror for ISO") selectmirror && REPLY= ;;
                "Sabayon_Linux_DAILY_amd64_ForensicsXfce") MESG ;;
                "Sabayon_Linux_DAILY_amd64_GNOME") MESG ;;
                "Sabayon_Linux_DAILY_amd64_GNOME-dev") MESG  ;;
                "Sabayon_Linux_DAILY_amd64_KDE") MESG ;;
                "Sabayon_Linux_DAILY_amd64_KDE-dev") MESG ;;
                "Sabayon_Linux_DAILY_amd64_LXQt") MESG ;;
                "Sabayon_Linux_DAILY_amd64_LXQt-dev") MESG  ;;
                "Sabayon_Linux_DAILY_amd64_MATE") MESG ;;
                "Sabayon_Linux_DAILY_amd64_MATE-dev") MESG ;;
                "Sabayon_Linux_DAILY_amd64_Minimal") MESG ;;
                "Sabayon_Linux_DAILY_amd64_Minimal-dev") MESG ;;
                "Sabayon_Linux_DAILY_amd64_Server") MESG ;;
                "Sabayon_Linux_DAILY_amd64_Server-dev") MESG ;;
                "Sabayon_Linux_DAILY_amd64_SpinBase") MESG ;;
                "Sabayon_Linux_DAILY_amd64_SpinBase-dev") MESG ;;
                "Sabayon_Linux_DAILY_amd64_Xfce") MESG  ;;
                "Sabayon_Linux_DAILY_amd64_Xfce-dev") MESG  ;;
                "Sabayon_Linux_DAILY_amd64_tarball.tar.gz") GETTAR  ;;
                "Quit") $CLEAR
                            echo "Thanks for using Sabayon Sync ISO"
                            $SLEEP 5
                            break ;;
                *) echo -e $LRED "invalid option" ;;
            esac
            echo -e $DEF
        done
nomorelogic
Simple Hen
 
Posts: 68
Joined: Sat Jan 14, 2012 13:40

Next

Return to ISO Testing & Molecule

Who is online

Users browsing this forum: No registered users and 1 guest

cron