improve testing documentation

diff --git a/README.markdown b/README.markdown
index 715975fade096ae83d1b73a04fee0855a84173d9..95ef1c2834986bf5deb0dce37514777d33457619 100644 (file)
--- a/README.markdown
+++ b/README.markdown
@@ -19,6 +19,8 @@ To add or modify a script do the following:
 8. Review the travis report once it is done.
 9. Await review of your changes by one of our developers.
 
+Optionally, to run tests locally, see more information under [_testing](_testing).
+
 ### Version Numbering
 
 To increase the version numbering, take the following conditions into account:

diff --git a/_testing/README.markdown b/_testing/README.markdown
index 6c06001a2142723f09fc1dca842d31daaaeb52a3..c3480bce27f9aba9d65cdc0ae51f791b2543e524 100644 (file)
--- a/_testing/README.markdown
+++ b/_testing/README.markdown
@@ -8,12 +8,30 @@ Main test runner is run-test.zsh. These tests are done:
 * Try to load the script in irssi
 * Check perlcritic report
 
+To run tests yourself, you first need several programs installed:
+* [zsh](http://zsh.sourceforge.net/)
+* [irssi](https://irssi.org)
+* cpan [Perl::Critic](https://metacpan.org/pod/Perl::Critic)
+* cpan [Perl::PrereqScanner](https://metacpan.org/pod/Perl::PrereqScanner)
+* cpan -f [Tree::XPathEngine](https://metacpan.org/pod/Tree::XPathEngine)
+* cpan [PPIx::XPath](https://metacpan.org/pod/PPIx::XPath)
+
+Then, run it like this:
+
+    ./_testing/run-test.zsh yourscript
+
+One "." should be printed.
+
 Evaluation of test success is done in report-test.zsh. Currently the
 following criteria lead to fail:
 * Script doesn't compile/load
 * Script doesn't use strict; or uses two-arg "open"
 * Script doesn't define %IRSSI and $VERSION
 
+To see the test results, run:
+
+    ./_testing/report-test.zsh yourscript
+
 The output table is as follows:
 - LOAD:  did the script compile/load successfully?
 - HDR:   was %IRSSI and $VERSION given?
@@ -29,9 +47,17 @@ definition *if* the script compiled cleanly. This can be used as a
 guidance for reviewers, but a lot of perl "critic" is stupid and a
 style question only.
 
+To see the detailed output, run:
+
+    ./_testing/travis/show-failures.zsh yourscript
+
 Errors and warnings visible in the Irssi log can serve as further
 pointers to both authors and reviewers.
 
+You can also inspect the raw outputs in the following folder:
+
+    ./Test/yourscript
+
 The following keys are recognised in config.yml:
 
 * cpan:
@@ -41,3 +67,13 @@ The following keys are recognised in config.yml:
                       because they hang Travis
 * whitelist: - list of scripts that are allowed to fail
 * scripts_yaml_keys: - list of keys to copy from irssi header to scripts.dmp
+
+To manually generate the _data/scripts.yaml file, run:
+
+    perl ./_testing/travis/update-scripts-yaml.pl
+
+To download the cached test results, run:
+
+    git fetch origin ci-artefacts:ci-artefacts
+    ln -s ci-artefacts Test
+

diff --git a/_testing/_irssi_test.pl b/_testing/_irssi_test.pl
index 462d86b74ae8fbdd3b753dd980305e322da0405f..4738d02b070a1f472e04daefdcae7aba866afcae 100644 (file)
--- a/_testing/_irssi_test.pl
+++ b/_testing/_irssi_test.pl
@@ -13,7 +13,6 @@ my $PWD = $ENV{PWD};
 my $SWD = "$PWD/../..";
 Irssi::command('^window log on');
 Irssi::command("script load $CURRENT_SCRIPT");
-Irssi::command('^window log off');
 
 my (@packages) = grep { !/^_/ } keys %Irssi::Script::;
 my $tp = $CURRENT_SCRIPT; $tp =~ s/^.*\///; $tp =~ s/\W/_/g; my @tmp;
@@ -98,3 +97,4 @@ $info{modules} = \@modules if @modules;
 $info{commands} = \@commands if @commands;
 $info{default_package} = $package =~ s/::$//r if $package;
 YAML::Tiny::DumpFile("info.yml", [\%info]);
+Irssi::command('^window log off');

diff --git a/_testing/report-test.zsh b/_testing/report-test.zsh
index 27e6b8351fc7fe1c7fa7c72a2c6450eeee66557e..d3217ed00bb2e7da259fb4f1f692c392d5bd29a9 100755 (executable)
--- a/_testing/report-test.zsh
+++ b/_testing/report-test.zsh
@@ -40,12 +40,23 @@ for scriptfile ($filelist) {
     if [[ $MARKDOWN_REPORT == 1 ]] { print -n '](Test/'$scriptfile:t:r'/)' }
     print -n $T
     local pass=0
+
+    # LOAD
     if [[ -f "Test/${scriptfile:t:r}/failed.yml" ]] { print -n '  '$failmark'   ' } \
-    else { print -n '  '$passmark'   '; ((++pass)) }; print -n $T
+    elif [[ -f "Test/${scriptfile:t:r}/info.yml" ]] { print -n '  '$passmark'   '; ((++pass)) } \
+    else { print -n '  '$skipmark'   '; }; print -n $T
+
+    # HDR
     if { grep -qs 'Severity: 6' "Test/${scriptfile:t:r}/perlcritic.log" } { print -n '  '$failmark'   ' } \
+    elif { grep -qs 'command not found:' "Test/${scriptfile:t:r}/perlcritic.log" } { print -n '  '$skipmark'   ' } \
     else { print -n '  '$passmark'   '; ((++pass)) }; print -n $T
+
+    # CRIT
     if { grep -qs 'Code before strictures are enabled\|Two-argument "open" used' "Test/${scriptfile:t:r}/perlcritic.log" }  { print -n '  '$failmark'   ' } \
+    elif { grep -qs 'command not found:' "Test/${scriptfile:t:r}/perlcritic.log" } { print -n '  '$skipmark'   ' } \
     else { print -n '  '$passmark'   '; ((++pass)) }; print -n $T
+
+    # SCORE
     perl -ne '$score += $1 -1 if /Severity: (\d+)/; END { printf "%3d", $score }' "Test/${scriptfile:t:r}/perlcritic.log" 2>/dev/null
     print -n '   '$T
     if [[ $pass -lt 3 ]]  {

diff --git a/_testing/run-test.zsh b/_testing/run-test.zsh
index ac34c068b284b6e24df7c00c0408978c0b5d6d42..5a508f52697238c06dbda01485c08cc763ec6c9f 100755 (executable)
--- a/_testing/run-test.zsh
+++ b/_testing/run-test.zsh
@@ -1,5 +1,4 @@
 #!/bin/zsh
-mkdir -p Test
 local base_path="`pwd`"
 local test_script="$base_path/_testing/_irssi_test.pl"
 
@@ -7,8 +6,8 @@ local test_script="$base_path/_testing/_irssi_test.pl"
 
 for scriptfile ($filelist) {
     rm -rf "Test/${scriptfile:t:r}"
-    mkdir "Test/${scriptfile:t:r}"
-    perlcritic --theme certrule --exclude RequireEndWithOne -2 $scriptfile >"Test/${scriptfile:t:r}/perlcritic.log"
+    mkdir -p "Test/${scriptfile:t:r}"
+    perlcritic --theme certrule --exclude RequireEndWithOne -2 $scriptfile >"Test/${scriptfile:t:r}/perlcritic.log" 2>&1
     pushd Test
     rm -fr .home
     mkdir .home