Merge branch 'shprdchris/create-raw-tests' into 'master'

Add CI testing for raw files

See merge request CodethinkLabs/lorry/lorry!37

7 files changed, 205 insertions, 1 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fd377a2..970c34f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -21,8 +21,12 @@ test:
     # Deps for running tests
     - apt-get install -y -qq cmdtest
 
+    # Download git-lfs repo
+    - apt-get install -y -qq curl
+    - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
+
     # Deps to run lorry tests
-    - apt-get install -y -qq git bzr cvs mercurial subversion hg-fast-export git-svn git-cvs bzr-fastimport
+    - apt-get install -y -qq git bzr cvs mercurial subversion hg-fast-export git-svn git-cvs git-lfs bzr-fastimport
     - pip3 install pyyaml
     - pip3 install https://gitlab.com/trovekube/cliapp/-/archive/cliapp-1.20180812.1/cliapp-cliapp-1.20180812.1.tar.gz
 
diff --git a/tests/raw-multiple-files.script b/tests/raw-multiple-files.script
new file mode 100755
index 0000000..4c16c51
--- /dev/null
+++ b/tests/raw-multiple-files.script
@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# Tests converting a folder with three raw files into a git repository.
+# Verifies that updating the relative paths in the .lorry file and
+# running again updates the relative paths in the git repo.
+#
+# Copyright (C) 2022 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+logfile="$DATADIR/raw-multiple-files.log"
+workdir="$DATADIR/work-dir"
+
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/raw-multiple-files.lorry" > /dev/null
+
+# update the lorry file, verify relative paths in git repo have changed
+sed -i 's/subdir_two/./g' ${DATADIR}/raw-multiple-files.lorry
+
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/raw-multiple-files.lorry" > /dev/null
+
+# verify that the git repositories were created successfully
+cd "$workdir/multiple-files-repo/git-a"
+
+# verify the files were created correctly the first time
+git ls-tree -r --name-only HEAD
+
+cd "$workdir/multiple-files-repo/git-b/"
+
+# verify the files have been updated correctly
+git ls-tree -r --name-only HEAD
+
+# list the commit messages
+git log --pretty='%s' master | sed -e"s,${DATADIR},DATADIR,"
diff --git a/tests/raw-multiple-files.setup b/tests/raw-multiple-files.setup
new file mode 100755
index 0000000..7e4aa12
--- /dev/null
+++ b/tests/raw-multiple-files.setup
@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Creates three raw files, and a lorry file specifying their downstream
+# subdirectories.
+#
+# Copyright (C) 2022 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+set -e
+
+# create the raw file directory
+repo="$DATADIR/raw-multiple-files-repo"
+mkdir "$repo"
+echo "first line, test 1" > "$repo/test_1.txt"
+echo "first line, test 2" > "$repo/test_2.txt"
+echo "first line, test 3" > "$repo/test_3.txt"
+
+# create the .lorry file for the raw file
+cat <<EOF > $DATADIR/raw-multiple-files.lorry
+{
+    "multiple-files-repo": {
+        "type": "raw-file",
+        "urls": [
+            {
+                "url": "file://$repo/test_1.txt",
+                "destination": "subdir_one"
+            },
+            {
+                "url": "file://$repo/test_2.txt",
+                "destination": "subdir_one"
+            },
+            {
+                "url": "file://$repo/test_3.txt",
+                "destination": "subdir_two"
+            }
+        ]
+    }
+}
+EOF
+
+# create the working directory
+test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir"
diff --git a/tests/raw-multiple-files.stdout b/tests/raw-multiple-files.stdout
new file mode 100644
index 0000000..1a55aa5
--- /dev/null
+++ b/tests/raw-multiple-files.stdout
@@ -0,0 +1,14 @@
+.gitattributes
+subdir_one/test_1.txt
+subdir_one/test_2.txt
+subdir_two/test_3.txt
+.gitattributes
+subdir_one/test_1.txt
+subdir_one/test_2.txt
+test_3.txt
+Remove redundant file paths.
+import test_3.txt
+import test_3.txt
+import test_2.txt
+import test_1.txt
+Ensure LFS is configured
diff --git a/tests/raw-single-file.script b/tests/raw-single-file.script
new file mode 100755
index 0000000..b46e0c1
--- /dev/null
+++ b/tests/raw-single-file.script
@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# Tests converting a folder with one raw file to a git repository.
+#
+# Copyright (C) 2022 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+set -e
+
+logfile="$DATADIR/raw-single-file.log"
+workdir="$DATADIR/work-dir"
+
+"${SRCDIR}/test-lorry" --pull-only --log="$logfile" --working-area="$workdir" \
+  "$DATADIR/raw-single-file.lorry" > /dev/null
+
+# verify that the git repository was created successfully
+cd "$workdir/single-file-repo/git-a/"
+
+# verify the file has been added correctly
+git ls-tree -r --name-only HEAD
+
+# list the commit messages
+git log --pretty='%s' master | sed -e"s,${DATADIR},DATADIR,"
diff --git a/tests/raw-single-file.setup b/tests/raw-single-file.setup
new file mode 100755
index 0000000..dde24c1
--- /dev/null
+++ b/tests/raw-single-file.setup
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# Creates a raw file and corresponding lorry file.
+#
+# Copyright (C) 2022 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+set -e
+
+# create the raw file directory
+repo="$DATADIR/raw-single-file-repo"
+mkdir "$repo"
+echo "first line" > "$repo/test.txt"
+
+# create the .lorry file for the raw file
+cat <<EOF > $DATADIR/raw-single-file.lorry
+{
+    "single-file-repo": {
+        "type": "raw-file",
+        "urls": [
+            {
+                "url": "file://$repo/test.txt",
+                "destination": "subdir_test"
+            }
+        ]
+    }
+}
+EOF
+
+# create the working directory
+test -d "$DATADIR/work-dir" || mkdir "$DATADIR/work-dir"
diff --git a/tests/raw-single-file.stdout b/tests/raw-single-file.stdout
new file mode 100644
index 0000000..fe73f17
--- /dev/null
+++ b/tests/raw-single-file.stdout
@@ -0,0 +1,4 @@
+.gitattributes
+subdir_test/test.txt
+import test.txt
+Ensure LFS is configured