b=444050

p=rhelmer
r=ccooper
add unit tests for make_incremental_updates.py (NPOTB)


git-svn-id: svn://10.0.0.236/trunk@258307 18797224-902f-48f8-a5cc-f745e15eee43

mozilla/tools/update-packaging/test/buildrefmars.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# Builds all the reference mars
+
+rm ref.mar
+rm ref-mac.mar
+
+ ../make_incremental_update.sh ref.mar `pwd`/from `pwd`/to
+ ../make_incremental_update.sh ref-mac.mar `pwd`/from `pwd`/to-mac
+
+rm product-1.0.lang.platform.complete.mar
+rm product-2.0.lang.platform.complete.mar
+rm product-2.0.lang.mac.complete.mar
+
+./make_full_update.sh product-1.0.lang.platform.complete.mar `pwd`/from
+./make_full_update.sh product-2.0.lang.platform.complete.mar `pwd`/to 
+./make_full_update.sh product-2.0.lang.mac.complete.mar `pwd`/to-mac 

mozilla/tools/update-packaging/test/catmanifest.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# helper tool for testing.  Cats the manifest out of a mar file
+
+mar="$1"
+workdir="/tmp/catmanifest"
+
+rm -rf "$workdir"
+mkdir -p "$workdir"
+cp "$1" "$workdir"
+cd "$workdir"
+mar -x "$1"
+mv update.manifest update.manifest.bz2
+bzip2 -d update.manifest.bz2
+cat update.manifest

mozilla/tools/update-packaging/test/common.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+#
+# Code shared by update packaging scripts.
+# Author: Darin Fisher
+#
+# In here to use the local common.sh to allow the full mars to have unfiltered files
+
+# -----------------------------------------------------------------------------
+# By default just assume that these tools exist on our path
+MAR=${MAR:-mar}
+BZIP2=${BZIP2:-bzip2}
+MBSDIFF=${MBSDIFF:-mbsdiff}
+
+# -----------------------------------------------------------------------------
+# Helper routines
+
+notice() {
+  echo $* 1>&2
+}
+
+get_file_size() {
+  info=($(ls -ln "$1"))
+  echo ${info[4]}
+}
+
+copy_perm() {
+  reference="$1"
+  target="$2"
+
+  if [ -x "$reference" ]; then
+    chmod 0755 "$target"
+  else
+    chmod 0644 "$target"
+  fi
+}
+
+make_add_instruction() {
+  f="$1"
+  is_extension=$(echo "$f" | grep -c 'extensions/.*/')
+  if [ $is_extension = "1" ]; then
+    # Use the subdirectory of the extensions folder as the file to test
+    # before performing this add instruction.
+    testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    echo "add-if \"$testdir\" \"$f\""
+  else
+    echo "add \"$f\""
+  fi
+}
+
+make_patch_instruction() {
+  f="$1"
+  is_extension=$(echo "$f" | grep -c 'extensions/.*/')
+  is_search_plugin=$(echo "$f" | grep -c 'searchplugins/.*')
+  if [ $is_extension = "1" ]; then
+    # Use the subdirectory of the extensions folder as the file to test
+    # before performing this add instruction.
+    testdir=$(echo "$f" | sed 's/\(extensions\/[^\/]*\)\/.*/\1/')
+    echo "patch-if \"$testdir\" \"$f.patch\" \"$f\""
+  elif [ $is_search_plugin = "1" ]; then
+    echo "patch-if \"$f\" \"$f.patch\" \"$f\""
+  else
+    echo "patch \"$f.patch\" \"$f\""
+  fi
+}
+
+append_remove_instructions() {
+  dir="$1"
+  if [ -f "$dir/removed-files" ]; then
+    prefix=
+    listfile="$dir/removed-files"
+  elif [ -f "$dir/Contents/MacOS/removed-files" ]; then
+    prefix=Contents/MacOS/
+    listfile="$dir/Contents/MacOS/removed-files"
+  fi
+  if [ -n "$listfile" ]; then
+    # Map spaces to pipes so that we correctly handle filenames with spaces.
+    files=($(cat "$listfile" | tr " " "|"))  
+    num_files=${#files[*]}
+    for ((i=0; $i<$num_files; i=$i+1)); do
+      # Trim whitespace (including trailing carriage returns)
+      f=$(echo ${files[$i]} | tr "|" " " | sed 's/^ *\(.*\) *$/\1/' | tr -d '\r')
+      # Exclude any blank lines or any lines ending with a slash, which indicate
+      # directories.  The updater doesn't know how to remove entire directories.
+      if [ -n "$f" ]; then
+        if [ $(echo "$f" | grep -c '\/$') = 0 ]; then
+          echo "remove \"$prefix$f\""
+        else
+          notice "ignoring remove instruction for directory: $f"
+        fi
+      fi
+    done
+  fi
+}
+
+# List all files in the current directory, stripping leading "./"
+# Skip the channel-prefs.js file as it should not be included in any
+# generated MAR files (see bug 306077). Pass a variable name and it will be
+# filled as an array.
+list_files() {
+  count=0
+
+  # Schrep - removed the exclusion cases here to allow for generation
+  # of testing mars
+  find . -type f \
+    | sed 's/\.\/\(.*\)/\1/' \
+    | sort > "$workdir/temp-filelist"
+  while read file; do
+    eval "${1}[$count]=\"$file\""
+    (( count++ ))
+  done < "$workdir/temp-filelist"
+  rm "$workdir/temp-filelist"
+}

mozilla/tools/update-packaging/test/diffmar.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Compares two mars
+
+marA="$1"
+marB="$2"
+workdir="/tmp/diffmar"
+fromdir="$workdir/0"
+todir="$workdir/1"
+
+rm -rf "$workdir"
+mkdir -p "$fromdir"
+mkdir -p "$todir"
+
+cp "$1" "$fromdir"
+cp "$2" "$todir"
+
+cd "$fromdir"
+mar -x "$1"
+rm "$1"
+mv update.manifest update.manifest.bz2
+bzip2 -d update.manifest.bz2
+ls -algR > files.txt
+# Sort the manifest so we don't get any diffs for ordering
+#cat update.manifest | sort > update.manifest
+
+cd "$todir"
+mar -x "$2"
+rm "$2"
+mv update.manifest update.manifest.bz2
+bzip2 -d update.manifest.bz2
+# Sort the manifest so we don't get any diffs for ordering
+#cat update.manifest | sort > update.manifest
+ls -algR > files.txt
+
+diff -r "$fromdir" "$todir"

mozilla/tools/update-packaging/test/from/channel-prefs.js

@@ -0,0 +1 @@
+from file shouldn't go in update

mozilla/tools/update-packaging/test/from/diff-patch-larger-than-file.txt

@@ -0,0 +1 @@
+from file

mozilla/tools/update-packaging/test/from/extensions/diff/diff-patch-larger-than-file.txt

@@ -0,0 +1 @@
+from file

mozilla/tools/update-packaging/test/from/force.txt

@@ -0,0 +1 @@
+file is same

mozilla/tools/update-packaging/test/from/readme.txt

@@ -0,0 +1,2 @@
+This from file should be ignored
+

mozilla/tools/update-packaging/test/from/removed.txt

@@ -0,0 +1 @@
+removed

mozilla/tools/update-packaging/test/from/same.txt

@@ -0,0 +1 @@
+file is same

mozilla/tools/update-packaging/test/from/searchplugins/diff/diff-patch-larger-than-file.txt

@@ -0,0 +1 @@
+from file

mozilla/tools/update-packaging/test/from/update.manifest

@@ -0,0 +1 @@
+from file shouldn't go in update

mozilla/tools/update-packaging/test/from/{foodir/channel-prefs.js

@@ -0,0 +1 @@
+from file shouldn't go in update

mozilla/tools/update-packaging/test/from/{foodir/diff-patch-larger-than-file.txt

@@ -0,0 +1 @@
+from file

mozilla/tools/update-packaging/test/from/{foodir/force.txt

@@ -0,0 +1 @@
+file is same

mozilla/tools/update-packaging/test/from/{foodir/readme.txt

@@ -0,0 +1 @@
+This from file should be ignored

mozilla/tools/update-packaging/test/from/{foodir/removed.txt

@@ -0,0 +1 @@
+removed

mozilla/tools/update-packaging/test/from/{foodir/same.txt

@@ -0,0 +1 @@
+file is same

mozilla/tools/update-packaging/test/from/{foodir/update.manifest

@@ -0,0 +1 @@
+from file shouldn't go in update

mozilla/tools/update-packaging/test/make_full_update.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# This tool generates full update packages for the update system.
+# Author: Darin Fisher
+#
+# In here to use the local common.sh to allow the full mars to have unfiltered files
+
+. $(dirname "$0")/common.sh
+
+# -----------------------------------------------------------------------------
+
+print_usage() {
+  notice "Usage: $(basename $0) [OPTIONS] ARCHIVE DIRECTORY"
+}
+
+if [ $# = 0 ]; then
+  print_usage
+  exit 1
+fi
+
+if [ $1 = -h ]; then
+  print_usage
+  notice ""
+  notice "The contents of DIRECTORY will be stored in ARCHIVE."
+  notice ""
+  notice "Options:"
+  notice "  -h  show this help text"
+  notice ""
+  exit 1
+fi
+
+# -----------------------------------------------------------------------------
+
+archive="$1"
+targetdir="$2"
+workdir="$targetdir.work"
+manifest="$workdir/update.manifest"
+targetfiles="update.manifest"
+
+mkdir -p "$workdir"
+
+# Generate a list of all files in the target directory.
+pushd "$targetdir"
+if test $? -ne 0 ; then
+  exit 1
+fi
+
+list_files files
+
+popd
+
+> $manifest
+
+num_files=${#files[*]}
+
+for ((i=0; $i<$num_files; i=$i+1)); do
+  f="${files[$i]}"
+
+  notice "processing $f"
+
+  make_add_instruction "$f" >> $manifest
+
+  dir=$(dirname "$f")
+  mkdir -p "$workdir/$dir"
+  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  copy_perm "$targetdir/$f" "$workdir/$f"
+
+  targetfiles="$targetfiles \"$f\""
+done
+
+# Append remove instructions for any dead files.
+append_remove_instructions "$targetdir" >> $manifest
+
+$BZIP2 -z9 "$manifest" && mv -f "$manifest.bz2" "$manifest"
+
+eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
+mv -f "$workdir/output.mar" "$archive"
+
+# cleanup
+rm -fr "$workdir"

mozilla/tools/update-packaging/test/runtests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+echo "testing make_incremental_updates.py"
+python ../make_incremental_updates.py -f testpatchfile.txt
+
+echo "diffing ref.mar and test.mar"
+./diffmar.sh ref.mar test.mar
+echo "diffing ref-mac.mar and test-mac.mar"
+./diffmar.sh ref-mac.mar test-mac.mar
+
+
+echo "testing make_incremental_updates_mar.py"
+python ../make_incremental_updates_mar.py -f testpatchfile.txt
+
+echo "diffing ref.mar and test.mar"
+./diffmar.sh ref.mar test.mar
+echo "diffing ref-mac.mar and test-mac.mar"
+./diffmar.sh ref-mac.mar test-mac.mar
+

mozilla/tools/update-packaging/test/testpatchfile.txt

@@ -0,0 +1,2 @@
+product-1.0.lang.platform.complete.mar,product-2.0.lang.platform.complete.mar,test.mar,""
+product-1.0.lang.platform.complete.mar,product-2.0.lang.mac.complete.mar,test-mac.mar,""

mozilla/tools/update-packaging/test/to-mac/Contents/MacOS/removed-files

@@ -0,0 +1,13 @@
+removed1.txt
+removed2.bin
+removed3-foo.txt     
+dir/
+this file has spaces
+notherdir/
+
+
+extra-spaces    
+
+lastFile
+
+

mozilla/tools/update-packaging/test/to/addFeedPrefs.js

@@ -0,0 +1 @@
+this is a new file

mozilla/tools/update-packaging/test/to/added.txt

@@ -0,0 +1 @@
+added file

mozilla/tools/update-packaging/test/to/channel-prefs.js

@@ -0,0 +1 @@
+to file shouldn't go in update

mozilla/tools/update-packaging/test/to/diff-patch-larger-than-file.txt

@@ -0,0 +1 @@
+file to

mozilla/tools/update-packaging/test/to/extensions/added/file.txt

@@ -0,0 +1 @@
+extfile

mozilla/tools/update-packaging/test/to/extensions/diff/diff-patch-larger-than-file.txt

@@ -0,0 +1 @@
+to file

mozilla/tools/update-packaging/test/to/force.txt

@@ -0,0 +1 @@
+file is same

mozilla/tools/update-packaging/test/to/readme.txt

@@ -0,0 +1 @@
+This to file should be ignored

mozilla/tools/update-packaging/test/to/removed-files

@@ -0,0 +1,13 @@
+removed1.txt
+removed2.bin
+removed3-foo.txt     
+dir/
+this file has spaces
+notherdir/
+
+
+extra-spaces    
+
+lastFile
+
+

mozilla/tools/update-packaging/test/to/same.txt

@@ -0,0 +1 @@
+file is same

mozilla/tools/update-packaging/test/to/searchplugins/added/file.txt

@@ -0,0 +1 @@
+extfile

mozilla/tools/update-packaging/test/to/searchplugins/diff/diff-patch-larger-than-file.txt

@@ -0,0 +1 @@
+to file

mozilla/tools/update-packaging/test/to/update.manifest

@@ -0,0 +1 @@
+to file shouldn't go in update

mozilla/tools/update-packaging/test/to/{foodir/added.txt

@@ -0,0 +1 @@
+added file

mozilla/tools/update-packaging/test/to/{foodir/channel-prefs.js

@@ -0,0 +1 @@
+to file shouldn't go in update

mozilla/tools/update-packaging/test/to/{foodir/diff-patch-larger-than-file.txt

@@ -0,0 +1 @@
+file to

mozilla/tools/update-packaging/test/to/{foodir/force.txt

@@ -0,0 +1 @@
+file is same

mozilla/tools/update-packaging/test/to/{foodir/readme.txt

@@ -0,0 +1 @@
+This to file should be ignored

mozilla/tools/update-packaging/test/to/{foodir/same.txt

@@ -0,0 +1 @@
+file is same

mozilla/tools/update-packaging/test/to/{foodir/update.manifest

@@ -0,0 +1 @@
+to file shouldn't go in update

mozilla/tools/update-packaging/test_make_incremental_updates.py

@@ -0,0 +1,118 @@
+#!/usr/bin/python
+
+import unittest
+import make_incremental_updates as mkup
+from make_incremental_updates import PatchInfo, MarFileEntry
+
+class TestPatchInfo(unittest.TestCase):
+    def setUp(self):
+        self.work_dir = 'work_dir'
+        self.file_exclusion_list = ['channel-prefs.js','update.manifest','removed-files']
+        self.path_exclusion_list = ['/readme.txt']
+        self.patch_info = PatchInfo(self.work_dir, self.file_exclusion_list, self.path_exclusion_list)
+
+    def testPatchInfo(self):
+        self.assertEquals(self.work_dir, self.patch_info.work_dir)
+        self.assertEquals([], self.patch_info.archive_files)
+        self.assertEquals([], self.patch_info.manifest)
+        self.assertEquals(self.file_exclusion_list, self.patch_info.file_exclusion_list)
+        self.assertEquals(self.path_exclusion_list, self.patch_info.path_exclusion_list)
+
+    def test_append_add_instruction(self):
+        self.patch_info.append_add_instruction('file.test')
+        self.assertEquals(['add "file.test"'], self.patch_info.manifest)
+
+    def test_append_patch_instruction(self):
+        self.patch_info.append_patch_instruction('file.test', 'patchname')
+        self.assertEquals(['patch "patchname" "file.test"'], self.patch_info.manifest)
+
+    """ FIXME touches the filesystem, need refactoring
+    def test_append_remove_instruction(self):
+        self.patch_info.append_remove_instruction('file.test')
+        self.assertEquals(['remove "file.test"'], self.patch_info.manifest)
+
+    def test_create_manifest_file(self):
+        self.patch_info.create_manifest_file()
+    """
+
+    def test_build_marfile_entry_hash(self):
+        self.assertEquals(({}, set([])), self.patch_info.build_marfile_entry_hash('root_path'))
+
+""" FIXME touches the filesystem, need refactoring
+class TestMarFileEntry(unittest.TestCase):
+    def setUp(self):
+        root_path = '.'
+        self.filename = 'file.test'
+        f = open(self.filename, 'w')
+        f.write('Test data\n')
+        f.close()
+        self.mar_file_entry = MarFileEntry(root_path, self.filename)
+
+    def test_calc_file_sha_digest(self):
+        f = open('test.sha', 'r')
+        goodSha = f.read()
+        f.close()
+        sha = self.mar_file_entry.calc_file_sha_digest(self.filename)
+        self.assertEquals(goodSha, sha)
+
+    def test_sha(self):
+        f = open('test.sha', 'r')
+        goodSha = f.read()
+        f.close()
+        sha = self.mar_file_entry.sha()
+        self.assertEquals(goodSha, sha)
+"""
+
+class TestMakeIncrementalUpdates(unittest.TestCase):
+    def setUp(self):
+        work_dir = '.'
+        self.patch_info = PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt'])
+        root_path = '/'
+        filename = 'test.file'
+        self.mar_file_entry = MarFileEntry(root_path, filename)
+
+    """ FIXME makes direct shell calls, need refactoring
+    def test_exec_shell_cmd(self):
+        mkup.exec_shell_cmd('echo test')
+
+    def test_copy_file(self):
+        mkup.copy_file('src_file_abs_path', 'dst_file_abs_path')
+
+    def test_bzip_file(self):
+        mkup.bzip_file('filename')
+
+    def test_bunzip_file(self):
+        mkup.bunzip_file('filename')
+
+    def test_extract_mar(self): 
+        mkup.extract_mar('filename', 'work_dir')
+
+    def test_create_partial_patch_for_file(self):
+        mkup.create_partial_patch_for_file('from_marfile_entry', 'to_marfile_entry', 'shas', self.patch_info)
+
+    def test_create_add_patch_for_file(self):           
+        mkup.create_add_patch_for_file('to_marfile_entry', self.patch_info)
+        
+    def test_process_explicit_remove_files(self): 
+        mkup.process_explicit_remove_files('dir_path', self.patch_info)
+    
+    def test_create_partial_patch(self):
+        mkup.create_partial_patch('from_dir_path', 'to_dir_path', 'patch_filename', 'shas', self.patch_info, 'forced_updates')
+
+    def test_create_partial_patches(patches):
+        mkup.create_partial_patches('patches')
+
+    """
+
+    """ FIXME touches the filesystem, need refactoring
+    def test_get_buildid(self):
+        mkup.get_buildid('work_dir', 'platform')
+    """
+
+    def test_decode_filename(self):
+        expected = {'locale': 'lang', 'platform': 'platform', 'product': 'product', 'version': '1.0', 'type': 'complete'}
+        self.assertEquals(expected, mkup.decode_filename('product-1.0.lang.platform.complete.mar'))
+        self.assertRaises(Exception, mkup.decode_filename, 'fail')
+       
+if __name__ == '__main__':
+    unittest.main()