Bug #384, fixes to Bash archiving script, add URL list file

README.md

@@ -39,7 +39,7 @@ Details of GDPR / privacy fixes can be found in [Bug #377][].
 
 * License: [GNU General Public License version 2][gpl].
 
-* See [CREDITS.txt][] for a list of the third-party libraries incorporated
+* See [CREDITS.md][] for a list of the third-party libraries incorporated
   in CloudEngine, and their authors and licenses.
 
 
@@ -53,7 +53,7 @@ Details of GDPR / privacy fixes can be found in [Bug #377][].
 [travis-icon]: https://travis-ci.org/IET-OU/cloudengine.svg
 [gpl]: https://gnu.org/licenses/gpl-2.0.html
 [license.txt]: https://github.com/IET-OU/cloudengine/blob/master/LICENCE.txt
-[credits.txt]: https://github.com/IET-OU/cloudengine/blob/master/CREDITS.txt
+[credits.md]: https://github.com/IET-OU/cloudengine/blob/master/CREDITS.md
 [cloudworks]: https://cloudworks.ac.uk/
 [iet]: https://iet.open.ac.uk/ "Developed by the Institute of Educational Technology"
 [ou]: https://www.open.ac.uk/

archive/cloudworks-archive.sh

@@ -5,12 +5,10 @@
 #
 # See: https://gist.github.com/steveosoule/79d0ba5f2cad558642aace43c7126946
 
-# wget --no-clobber --convert-links --random-wait -r -p --level 1 -E -e robots=off --user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36"  http://www.headstar.com/site/
-
 # wget --no-clobber --convert-links --random-wait -r -p --level 1 -E -e robots=off --user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:66.0) Gecko/20100101 Firefox/66.0" https://cloudworks.ac.uk/
 
 wget \
-  --level=8 \
+  --level=10 \
  	--mirror \
  	--recursive \
  	--execute robots=off \
@@ -22,10 +20,13 @@ wget \
  	--wait=1 \
  	--random-wait \
  	--domains cloudworks.ac.uk \
- 	--debug \
- 	--output-file=cloudworks.ac.uk-wget-2019-06-28.log \
- 	--progress=dot \
- 		https://cloudworks.ac.uk/
+	--debug \
+	--output-file=cloudworks.ac.uk-wget-2019-06-28--p2.log \
+	--progress=dot \
+	--show-progress \
+	--no-clobber \
+	--input-file=cloudworks-url-list.txt \
+		https://cloudworks.ac.uk/
 
 #  --directory-prefix=sample \
 

archive/cloudworks-url-list.txt

@@ -0,0 +1,107 @@
+
+https://cloudworks.ac.uk/
+
+https://cloudworks.ac.uk/user/view/1
+https://cloudworks.ac.uk/user/view/3
+https://cloudworks.ac.uk/user/following/3
+https://cloudworks.ac.uk/user/view/5
+https://cloudworks.ac.uk/user/view/6
+https://cloudworks.ac.uk/user/view/7
+https://cloudworks.ac.uk/user/view/8
+https://cloudworks.ac.uk/user/view/9
+
+https://cloudworks.ac.uk/user/view/10
+https://cloudworks.ac.uk/user/view/13
+https://cloudworks.ac.uk/user/view/15
+https://cloudworks.ac.uk/user/view/16
+https://cloudworks.ac.uk/user/view/20
+https://cloudworks.ac.uk/user/view/22
+https://cloudworks.ac.uk/user/view/28
+https://cloudworks.ac.uk/user/view/38
+https://cloudworks.ac.uk/user/view/66
+
+https://cloudworks.ac.uk/user/view/103
+https://cloudworks.ac.uk/user/view/104
+https://cloudworks.ac.uk/user/view/105
+https://cloudworks.ac.uk/user/view/108
+https://cloudworks.ac.uk/user/view/145
+https://cloudworks.ac.uk/user/view/158
+https://cloudworks.ac.uk/user/view/163
+https://cloudworks.ac.uk/user/view/202
+https://cloudworks.ac.uk/user/view/356
+https://cloudworks.ac.uk/user/view/361
+https://cloudworks.ac.uk/user/view/362
+https://cloudworks.ac.uk/user/view/363
+https://cloudworks.ac.uk/user/view/367
+https://cloudworks.ac.uk/user/view/725
+https://cloudworks.ac.uk/user/view/863
+https://cloudworks.ac.uk/user/view/929
+https://cloudworks.ac.uk/user/view/983
+https://cloudworks.ac.uk/user/following/983
+
+https://cloudworks.ac.uk/user/view/1003
+https://cloudworks.ac.uk/user/view/1031
+https://cloudworks.ac.uk/user/view/1040
+https://cloudworks.ac.uk/user/following/1040
+https://cloudworks.ac.uk/user/view/1055
+https://cloudworks.ac.uk/user/view/1126
+https://cloudworks.ac.uk/user/following/1126
+https://cloudworks.ac.uk/cloudscape/view/1879
+https://cloudworks.ac.uk/cloudscape/view/3004
+https://cloudworks.ac.uk/cloudscape/view/3026
+https://cloudworks.ac.uk/cloudscape/view/2005#!-CALRG-2010
+https://cloudworks.ac.uk/user/view/1230
+https://cloudworks.ac.uk/user/view/1789
+https://cloudworks.ac.uk/user/view/1926
+https://cloudworks.ac.uk/user/view/1933
+https://cloudworks.ac.uk/user/view/1943
+
+https://cloudworks.ac.uk/user/view/2033
+https://cloudworks.ac.uk/cloudscape/view/2043
+https://cloudworks.ac.uk/cloudscape/view/2065
+https://cloudworks.ac.uk/user/view/2294
+https://cloudworks.ac.uk/user/view/2403
+https://cloudworks.ac.uk/user/view/2651
+https://cloudworks.ac.uk/user/view/2702
+https://cloudworks.ac.uk/user/view/6651
+https://cloudworks.ac.uk/user/followers/6651
+
+https://cloudworks.ac.uk/user/view/1057
+
+https://cloudworks.ac.uk/cloudscape/view/387
+https://cloudworks.ac.uk/cloudscape/view/566
+https://cloudworks.ac.uk/cloud/view/20
+https://cloudworks.ac.uk/user/view/110
+https://cloudworks.ac.uk/cloud/view/55
+https://cloudworks.ac.uk/cloud/view/238
+
+https://cloudworks.ac.uk/user/view/1040
+https://cloudworks.ac.uk/cloudscape/view/1873
+https://cloudworks.ac.uk/cloudscape/view/2434
+https://cloudworks.ac.uk/tag/view/oEmbed
+https://cloudworks.ac.uk/cloud/view/5630#!-embed-test
+https://cloudworks.ac.uk/cloud/view/2207
+
+https://cloudworks.ac.uk/cloudscape/view/1959
+
+https://cloudworks.ac.uk/cloudscape/view/2899
+https://cloudworks.ac.uk/cloudscape/view/2916
+https://cloudworks.ac.uk/cloudscape/view/2945
+https://cloudworks.ac.uk/cloudscape/view/2994
+https://cloudworks.ac.uk/cloudscape/view/3017
+https://cloudworks.ac.uk/cloudscape/view/3030
+
+https://cloudworks.ac.uk/events/archive
+
+https://cloudworks.ac.uk/tag/view/altc2009
+https://cloudworks.ac.uk/cloudscape/view/1870
+https://cloudworks.ac.uk/cloudscape/followers/1870
+
+https://cloudworks.ac.uk/search/result/?q=test&x=0&y=0
+https://cloudworks.ac.uk/search/result/
+
+https://cloudworks.ac.uk/auth/register
+https://cloudworks.ac.uk/auth/login
+https://cloudworks.ac.uk/auth/forgotten_password
+
+# End.