diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..d48c469 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,4 @@ +shellcheck: + stage: build + script: + - /usr/bin/shellcheck -e SC2164,SC2086 share_lun_ui_fix.sh \ No newline at end of file diff --git a/README.md b/README.md index ca866d2..2fd6270 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # Share/LUN UI Fix Script This script finds shares that were accidentally broken in the UI and fixes it. +Additionally, performs migration of created folders in the roots of volumes to btrfs subvolume. ### Prerequisites -* You need to be on ReadyNAS OS 6.9.4 or newer. This script contains configuration file data for 6.9.4 that may or may not work with older versions. +* You should use this script with ReadyNAS OS 6.10.0 or newer. This script contains configuration file data for 6.10.0 that may or may not work with older versions of ReadyNASOS. * Either the ReadyNAS needs internet access or you need to copy/paste the data into a file on the NAS via SSH. ** For internet access, ensure you can reach the internet and DNS is working properly. * This does not work in `Tech Support mode`. You must be in `Normal mode` and `readynasd` needs to be running. @@ -44,58 +45,61 @@ root@readynasos:#/ bash -x /root/share_lun_ui_fix.sh ### Standard Output ``` root@rn524x:~# curl -s https://gitlab.codycook.us/readynas-scripts/share_lun_ui_fix.sh/raw/master/share_lun_ui_fix.sh | bash -=============================== -Share and iSCSI Rebuild v6.9.4 -=============================== -NOTE: This version of readynasd creates share backups in /var/backups/shares. - When possible, you should try and restore a valid backup config from this directory. - This share config gets rebuilt every time readynasd restarts. Because of this, we'll make a backup of the existing share backups before proceeding. -Generating backup config...... done! Backup complete! -============================================= -====== Listing Available Share Backups ====== -============================================= -total 1.3M -drwxr-x--- 1 root root 144 Oct 2 04:20 . -drwxr-xr-x 1 root root 866 Oct 2 10:31 .. --rw-r--r-- 1 root root 750K Oct 2 04:20 shares.tar.0 --rw-r--r-- 1 root root 124K Oct 1 04:20 shares.tar.1.gz --rw-r--r-- 1 root root 124K Sep 30 04:20 shares.tar.2.gz --rw-r--r-- 1 root root 124K Sep 29 04:20 shares.tar.3.gz --rw-r--r-- 1 root root 124K Sep 28 04:20 shares.tar.4.gz -============================================= -See the last modified dates and determine if you can restore from that config or not instead of using this script. -It is best to pick a date before the shares disappeared from readynasd. -============================================= -If you want to cancel and manually restore from one of these configs, CTRL-C now... -Otherwise, continuing in 10... 9... 8... 7... 6... 5... 4... 3... 2... 1... 0... -Querying readynasd for list of volumes... -4 volume(s) found. -Beginning in 3... 2... 1... -=============================== -=== All right! Time to go! === -=============================== -= Investigating volume /vol1... -= Objects found: fromharry -== Reviewing fromharry -=== Found share at /vol1/fromharry... recreating share configs.... done with /vol1/fromharry. -= done with /vol1 -= Investigating volume /vol2... -= Objects found: luntest -== Reviewing luntest -=== Found LUN at /vol2/luntest... recreating iSCSI configs.... done with /vol2/luntest. -= done with /vol2 -= Investigating volume /vol3... -No shares or LUNs found. -= done with /vol3 -= Investigating volume /vol4... -= Objects found: outrunner -== Reviewing outrunner -=== Found share at /vol4/outrunner... recreating share configs.... done with /vol4/outrunner. -= done with /vol4 -*** Restarting readynasd to finish changes... *** -================================================= -| Completed share creation! We were able to | -| rebuild 2 shares and 1 LUNs. | -================================================= +=================================== +* Share and iSCSI Rebuild v6.10.0 * +=================================== += [Info] Share config backups folder exist (/var/backups/shares) +== [Task] Backing up existing share config backup folder +=== [Info] Completed share config backup folder. +=== [Task] Listing backup folder for a quick analysis +2019-04-01 22:56:12.198942551 -0700 71680 /var/backups/shares/shares.tar.0 +2019-04-01 22:50:02.066740050 -0700 2058 /var/backups/shares/shares.tar.1.gz +2019-04-01 21:40:46.250729403 -0700 2012 /var/backups/shares/shares.tar.2.gz +2019-04-01 15:30:49.968039883 -0700 1754 /var/backups/shares/shares.tar.3.gz +2019-04-01 15:28:44.033709334 -0700 1753 /var/backups/shares/shares.tar.4.gz +==== [Info] Review backup folder listing and decide to continue or restore. +==== [Decide] To cancel and manually restore from an above config, CTRL-C now... Otherwise, continuing in 5... 4... 3... 2... 1... 0... += [Task] Discovering volumes +== [Info] 1 volume(s) found. +== [Task] Investigating volume /data... +=== [Info] Number of items found : 6 +=== [Info] Items found: Documents ImFake LUN1 Music Pictures Videos +=== [Task] Reviewing Documents +=== [Info] Found share at /data/Documents +=== [Task] Recreating share configs. +==== [Info] Done with Share /data/Documents. +=== [Task] Reviewing ImFake +=== [Task] Adding /data/ImFake to subvolume migration list. +=== [Task] Reviewing LUN1 +=== [Task] Recreating config for LUN at /data/LUN1.. +==== [Info] done with LUN /data/LUN1. +=== [Task] Reviewing Music +=== [Info] Found share at /data/Music +=== [Task] Recreating share configs. +==== [Info] Done with Share /data/Music. +=== [Task] Reviewing Pictures +=== [Info] Found share at /data/Pictures +=== [Task] Recreating share configs. +==== [Info] Done with Share /data/Pictures. +=== [Task] Reviewing Videos +=== [Info] Found share at /data/Videos +=== [Task] Recreating share configs. +==== [Info] Done with Share /data/Videos. +== [Task] Subvolume Migrations required: 1 +== [Info] Subvolume Migrations list: ImFake +=== [Warning] Do not interrupt this process. Doing so may result in an incomplete migration. +=== [Decide] If you want to terminate now, CTRL-C...Continuing in 5... 4... 3... 2... 1... 0... +=== [Task] Subvolume Migrating /data/ImFake +=== [Task] Creating replacement btrfs subvolume +==== [Task] Migrating data to the new subvolume +===== [Success] migrating /data/ImFake to a btrfs subvolume. +=== [Info] Found share at /data/ImFake +=== [Task] Recreating share configs. +=== [Info] done with /data/ImFake. += [Info] done with /data += [Task] Restarting readynasd to apply changes. +[Stats] Rebuilt Share configs: 5 +[Stats] Rebuilt LUN configs: 1 +[Stats] Migrated folders to subvolumes: 1 root@rn524x:~# ``` diff --git a/share_lun_ui_fix.sh b/share_lun_ui_fix.sh index 3c3cce6..cb1904f 100755 --- a/share_lun_ui_fix.sh +++ b/share_lun_ui_fix.sh @@ -2,18 +2,36 @@ # share fix # use this script to rebuild default configs for shares if the ._share becomes # broken for whatever reason. -unset readynasd number x volume found iscsi shares +unset readynasd number x volume found iscsi shares invalid verbosity volumes found=0 shares=0 iscsi=0 +migrated=0 +mvcmd="mv" +################ +### commands ### +################ + +backupsnappershots() { + echo "==== [Task] Backing up existing Snapper configs" + tar -cf "/var/backups/backup_snapper_configs.$(date +%s).tar" /etc/snapper/configs 2>/dev/null + if [[ $? ]]; then + echo "===== [Info] Success at backing up snapper configs" + echo "===== [Task] Removing Snapper Configs" + rm /etc/snapper/configs/* 2>/dev/null + else + echo "===== [Fail] Couldn't backup configs. Please debug" + fi +} share_create() { found=$((found+1)) shares=$((shares+1)) path="/$volume/._share/$item" - echo -n "=== Found share at /$volume/$item... recreating share configs...". + echo "=== [Info] Found share at /$volume/$item" + echo "=== [Task] Recreating share configs." mkdir -p "$path" echo "share" > "$path"/datasettype.conf - echo "0,0" > "$path"/recycle.conf + echo "0,25000" > "$path"/recycle.conf echo "#dashboard:available=0 \"/$volume/$item\" 127.0.0.1(insecure,insecure_locks,no_subtree_check,crossmnt,anonuid=99,anongid=99,root_squash,ro,async)" > "$path"/nfs.conf echo "" > "$path"/snapdir.conf echo "1 0 0 * * * 0" > "$path"/snapshot.conf @@ -48,7 +66,7 @@ refquota = 0 EOF cat > "$path"/ftp.conf << EOF available 0 -default_access disabled +default_access writeable mask fmask dmask @@ -62,52 +80,123 @@ EOF cat > "$path"/samba.conf << EOF [$item] path = /$volume/$item - comment = "" + comment = "$item" spotlight = 0 guest ok = 1 + force create mode = 0666 + create mask = 0666 + force directory mode = 1777 + directory mask = 1777 admin users = +admin writeable = 1 follow symlinks = 1 EOF - cd "$path" + + cat > /etc/snapper/configs/$((shares-1)) << EOF + +# subvolume to snapshot +SUBVOLUME="/$volume/$item" + +# filesystem type +FSTYPE="btrfs" + + +# btrfs qgroup for space aware cleanup algorithms +QGROUP="" + + +# fraction of the filesystems space the snapshots may use +SPACE_LIMIT="0.5" + + +# users and groups allowed to work with config +ALLOW_USERS="guest" +ALLOW_GROUPS="admin" + +# sync users and groups from ALLOW_USERS and ALLOW_GROUPS to .snapshots +# directory +SYNC_ACL="no" + + +# start comparing pre- and post-snapshot in background after creating +# post-snapshot +BACKGROUND_COMPARISON="yes" + + +# run daily number cleanup +NUMBER_CLEANUP="yes" + +# limit for number cleanup +NUMBER_MIN_AGE="1800" +NUMBER_LIMIT="50" +NUMBER_LIMIT_IMPORTANT="10" + + +# create hourly snapshots +TIMELINE_CREATE="yes" + +# cleanup hourly snapshots after some time +TIMELINE_CLEANUP="yes" + +# limits for timeline cleanup +TIMELINE_MIN_AGE="1800" +TIMELINE_LIMIT_HOURLY="10" +TIMELINE_LIMIT_DAILY="10" +TIMELINE_LIMIT_WEEKLY="0" +TIMELINE_LIMIT_MONTHLY="10" +TIMELINE_LIMIT_YEARLY="10" + + +# cleanup empty pre-post-pairs +EMPTY_PRE_POST_CLEANUP="yes" + +# limits for empty pre-post-pair cleanup +EMPTY_PRE_POST_MIN_AGE="1800" + +SNAPSHOT_HWM="13" +EOF + cd "$path" chown root.root datasettype.conf recycle.conf snapdir.conf snapshot.conf chown admin.admin afp.conf fs.conf ftp.conf nfs.conf samba.conf custom_snapshot_management.conf custom_snapshot_schedule.conf chmod 644 datasettype.conf recycle.conf snapdir.conf snapshot.conf afp.conf fs.conf ftp.conf nfs.conf samba.conf custom_snapshot_management.conf custom_snapshot_schedule.conf - echo " done with /$volume/$item." + chmod 640 /etc/snapper/configs/$((shares-1)) + echo "==== [Info] Done with Share /$volume/$item." } check_existing_backups() { if [ -d "/var/backups/shares/" ]; then - echo "NOTE: This version of readynasd creates share backups in /var/backups/shares." - echo " When possible, you should try and restore a valid backup config from this directory." - echo " This share config gets rebuilt every time readynasd restarts. Because of this, we'll make a backup of the existing share backups before proceeding." - sleep 3 - echo -n "Generating backup config..." && sleep 1 && tar -cf "/var/backups/backup_share_configs.$(date +%s).tar" /var/backups/shares 2>/dev/null && sleep 1 && echo "... done! Backup complete!" - sleep 3 - echo "=============================================" - echo "====== Listing Available Share Backups ======" - echo "=============================================" - sleep 2 - ls -lah /var/backups/shares/ + echo "= [Info] Share config backups folder exist (/var/backups/shares)" + echo "== [Task] Backing up existing share config backup folder" + tar -cf "/var/backups/backup_share_configs.$(date +%s).tar" /var/backups/shares 2>/dev/null + echo "=== [Info] Completed share config backup folder." sleep 1 - echo "=============================================" - echo "See the last modified dates and determine if you can restore from that config or not instead of using this script." - echo "It is best to pick a date before the shares disappeared from readynasd." - echo "=============================================" + echo "=== [Task] Listing backup folder for a quick analysis" + stat -c "%y %s %n" /var/backups/shares/* + echo "==== [Info] Review backup folder listing and decide to continue or restore." sleep 5 - echo "If you want to cancel and manually restore from one of these configs, CTRL-C now..." - echo -n "Otherwise, continuing in 10..."; sleep 1; echo -n " 9..."; sleep 1; echo -n " 8..."; sleep 1; echo -n " 7..."; sleep 1; echo -n " 6..."; sleep 1; echo -n " 5..."; sleep 1; echo -n " 4..."; sleep 1; echo -n " 3..."; sleep 1; echo -n " 2..."; sleep 1; echo -n " 1..."; sleep 1 ; echo " 0..."; sleep 1 + echo -n "==== [Decide] To cancel and manually restore from an above config, CTRL-C now... Otherwise, continuing in 5..."; + sleep 1; + echo -n " 4..."; + sleep 1; + echo -n " 3..."; + sleep 1; + echo -n " 2..."; + sleep 1; + echo -n " 1..."; + sleep 1 ; + echo " 0..."; + sleep 1 fi } iscsi_create() { iscsi=$((iscsi+1)) - found=$((found+1)) - path="/$volume/._share/$item" - echo -ne "=== Found LUN at /$volume/$item... recreating iSCSI configs...". + found=$((found+1)) + path="/$volume/._share/$item" + echo "=== [Task] Recreating config for LUN at /$volume/$item.." mkdir -p "$path" - cat > "$path"/iscsi.conf << EOF + cat > "$path"/iscsi.conf << EOF comment alert_threshold 80 EOF @@ -129,71 +218,112 @@ EOF ........................ EOF echo "0 0 0 * 0 0 0" > "$path"/snapshot.conf - echo " done with /$volume/$item." + echo "==== [Info] done with LUN /$volume/$item." } backup_configs() { -cd "/$volume/._share/" -tar -cf "/var/backups/$volume_sharelunconfig.$(date +%s).tar" ./* 2>/dev/null +cd "/$volume/._share/" +tar -cf "/var/backups/$volume.sharelunconfig.$(date +%s).tar" ./* 2>/dev/null } -echo "===============================" -echo "Share and iSCSI Rebuild v6.9.4" -echo "===============================" +checktype() { + if ls -QdA /"$volume"/"$item"/.iscsi >/dev/null 2>&1; then + iscsi_create + else + share_create + fi +} +################### +### end command ### +################### + + +echo "===================================" +echo "* Share and iSCSI Rebuild v6.10.0 *" +echo "===================================" check_existing_backups -echo "Querying readynasd for list of volumes..." - -x=$(rn_nml -g volumes 2>/dev/null) # find if readynasd is running - +echo "= [Task] Discovering volumes" +volumes=$(rn_nml -g volumes 2>/dev/null) # find if readynasd is running if [[ $? -eq 0 ]]; then # if readynasd was successful - x=$(echo "$x"| grep "resource-id" | sed 's/^.*id="//;s/".*//;') # then parse it + volumes=$(echo "$volumes"| grep "resource-id" | sed 's/^.*id="//;s/".*//;') # then parse it readynasd=1 elif [[ $? -eq 1 ]] ; then # but if it wasn't - echo "readynasd not available; doing alternative lookup for shares" # inform intent - x=$(find / -maxdepth 2 -name "._share" | sed 's/^\///g;s/\/.*//') # create new list + echo "== [Info] readynasd not available; doing alternative lookup for shares" # inform intent + volumes=$(find / -maxdepth 2 -name "._share" | sed 's/^\///g;s/\/.*//') # create new list fi - -number=$(echo "$x" | wc -w) - +number=$(echo "$volumes" | wc -w) if [[ ! $number ]]; then - echo "No volumes found!" + echo "== [Error] No volumes found." exit 1 else - echo "$number volume(s) found." - echo -n "Beginning in 3..."; sleep 1; echo -n " 2..."; sleep 1; echo " 1..."; sleep 1 - echo "===============================" - echo "=== All right! Time to go! ===" - echo "===============================" - for volume in $x; do - echo "= Investigating volume /$volume..." - objects=$(cd /"$volume"; ls -dA */ 2>/dev/null) + echo "== [Info] $number volume(s) found." + backupsnappershots + for volume in $volumes; do + echo "== [Task] Investigating volume /$volume..." + objects=$(cd /"$volume"; ls -dA ./*/ 2>/dev/null) if [[ $? == 2 ]]; then - echo "No shares or LUNs found." + echo "=== [Error] No shares or LUNs found." else - objects=$(echo "$objects" |egrep -v "^.apps\/$|^home\/$|^.purge\/$|^._share\/$|^.timemachine\/$|^.vault\/$|^.TemporaryItems\/$" | sed 's/\/$//') - echo "= Objects found:" $objects - backup_configs - for item in $objects; do - echo "== Reviewing $item" - if ls -QdA /"$volume"/"$item"/.iscsi >/dev/null 2>&1; then - iscsi_create - else - share_create - fi - done + objects=$(echo "$objects" |egrep -v "^.apps\/$|^\.\/home\/$|^.purge\/$|^._share\/$|^.timemachine\/$|^.vault\/$|^.TemporaryItems\/$" | sed 's/\/$//;s/\.\///') + echo "=== [Info] Number of items found : $(echo $objects | wc -w)" + echo "=== [Info] Items found:" $objects + backup_configs + for item in $objects; do + echo "=== [Task] Reviewing $item" + if ! btrfs subv show /"$volume"/"$item" &>/dev/null; then + echo "=== [Task] Adding /$volume/$item to subvolume migration list." + invalid="$invalid $item" + else + checktype + fi + done fi - echo "= done with /$volume" - done + if [[ $invalid ]]; then + invalidnumber=$(echo "$invalid" | wc -w) + echo "== [Task] Subvolume Migrations required: $invalidnumber" + echo "== [Info] Subvolume Migrations list: $invalid" + echo "=== [Warning] Do not interrupt this process. Doing so may result in an incomplete migration." + sleep 3 + echo -n "=== [Decide] If you want to terminate now, CTRL-C..." && echo -n "Continuing in 5..."; sleep 1; echo -n " 4..."; sleep 1; echo -n " 3..."; sleep 1; echo -n " 2..."; sleep 1; echo -n " 1..."; sleep 1 ; echo " 0..."; sleep 1 + for item in $invalid; do + echo "=== [Task] Subvolume Migrating /$volume/$item" + $mvcmd /"$volume"/"$item" /"$volume"/"$item".fix + if [[ ! $? ]]; then + echo "==== [Fail] when renaming folder." + elif [[ $? ]]; then + echo "=== [Task] Creating replacement btrfs subvolume" + btrfs subvolume create /"$volume"/"$item" &>/dev/null + if [[ $? ]]; then + echo "==== [Task] Migrating data to the new subvolume" + $mvcmd /"$volume"/"$item".fix/* /"$volume"/"$item".fix/.* /"$volume"/"$item" &>/dev/null + if [[ $? ]]; then + echo "===== [Success] migrating /$volume/$item to a btrfs subvolume." + rmdir /$volume/$item.fix + migrated=$((migrated+1)) + checktype + else + echo "===== [Error] Re-run in debug mode for more details." + fi + fi + else + echo "==== [Fail] when creating btrfs subvolume." + fi + done + + fi + echo "= [Info] done with /$volume" + unset invalid invalidnumber + done + if [[ $found -gt 0 ]] && [[ $readynasd ]] ; then - echo "*** Restarting readynasd to finish changes... ***" + echo "= [Task] Restarting readynasd to apply changes." systemctl restart readynasd else - echo "Not starting readynasd because it returned a bad message earlier; restart manually." + echo "= [Info] Not starting readynasd because it returned a bad message earlier; restart as needed." fi - echo "=================================================" - echo "| Completed share creation! We were able to |" - echo "| rebuild $shares shares and $iscsi LUNs. |" - echo "=================================================" + echo "[Stats] Rebuilt Share configs: $shares" + echo "[Stats] Rebuilt LUN configs: $iscsi" + echo "[Stats] Migrated folders to subvolumes: $migrated" fi