From: Ailin Nemui <redacted>
Date: Thu, 1 Feb 2018 15:24:53 +0000 (+0100)
Subject: improve testing documentation
X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=014b4289fc98a328412e5c81ce8aaa0ab57ef6db;p=irssi-scripts.irssi.org.git

improve testing documentation
---

diff --git a/README.markdown b/README.markdown
index 715975f..95ef1c2 100644
--- a/README.markdown
+++ b/README.markdown
@@ -19,6 +19,8 @@ To add or modify a script do the following:
 8. Review the travis report once it is done.
 9. Await review of your changes by one of our developers.
 
+Optionally, to run tests locally, see more information under [_testing](_testing).
+
 ### Version Numbering
 
 To increase the version numbering, take the following conditions into account:
diff --git a/_testing/README.markdown b/_testing/README.markdown
index 6c06001..c3480bc 100644
--- a/_testing/README.markdown
+++ b/_testing/README.markdown
@@ -8,12 +8,30 @@ Main test runner is run-test.zsh. These tests are done:
 * Try to load the script in irssi
 * Check perlcritic report
 
+To run tests yourself, you first need several programs installed:
+* [zsh](http://zsh.sourceforge.net/)
+* [irssi](https://irssi.org)
+* cpan [Perl::Critic](https://metacpan.org/pod/Perl::Critic)
+* cpan [Perl::PrereqScanner](https://metacpan.org/pod/Perl::PrereqScanner)
+* cpan -f [Tree::XPathEngine](https://metacpan.org/pod/Tree::XPathEngine)
+* cpan [PPIx::XPath](https://metacpan.org/pod/PPIx::XPath)
+
+Then, run it like this:
+
+    ./_testing/run-test.zsh yourscript
+
+One "." should be printed.
+
 Evaluation of test success is done in report-test.zsh. Currently the
 following criteria lead to fail:
 * Script doesn't compile/load
 * Script doesn't use strict; or uses two-arg "open"
 * Script doesn't define %IRSSI and $VERSION
 
+To see the test results, run:
+
+    ./_testing/report-test.zsh yourscript
+
 The output table is as follows:
 - LOAD:  did the script compile/load successfully?
 - HDR:   was %IRSSI and $VERSION given?
@@ -29,9 +47,17 @@ definition *if* the script compiled cleanly. This can be used as a
 guidance for reviewers, but a lot of perl "critic" is stupid and a
 style question only.
 
+To see the detailed output, run:
+
+    ./_testing/travis/show-failures.zsh yourscript
+
 Errors and warnings visible in the Irssi log can serve as further
 pointers to both authors and reviewers.
 
+You can also inspect the raw outputs in the following folder:
+
+    ./Test/yourscript
+
 The following keys are recognised in config.yml:
 
 * cpan:
@@ -41,3 +67,13 @@ The following keys are recognised in config.yml:
                       because they hang Travis
 * whitelist: - list of scripts that are allowed to fail
 * scripts_yaml_keys: - list of keys to copy from irssi header to scripts.dmp
+
+To manually generate the _data/scripts.yaml file, run:
+
+    perl ./_testing/travis/update-scripts-yaml.pl
+
+To download the cached test results, run:
+
+    git fetch origin ci-artefacts:ci-artefacts
+    ln -s ci-artefacts Test
+
diff --git a/_testing/_irssi_test.pl b/_testing/_irssi_test.pl
index 462d86b..4738d02 100644
--- a/_testing/_irssi_test.pl
+++ b/_testing/_irssi_test.pl
@@ -13,7 +13,6 @@ my $PWD = $ENV{PWD};
 my $SWD = "$PWD/../..";
 Irssi::command('^window log on');
 Irssi::command("script load $CURRENT_SCRIPT");
-Irssi::command('^window log off');
 
 my (@packages) = grep { !/^_/ } keys %Irssi::Script::;
 my $tp = $CURRENT_SCRIPT; $tp =~ s/^.*\///; $tp =~ s/\W/_/g; my @tmp;
@@ -98,3 +97,4 @@ $info{modules} = \@modules if @modules;
 $info{commands} = \@commands if @commands;
 $info{default_package} = $package =~ s/::$//r if $package;
 YAML::Tiny::DumpFile("info.yml", [\%info]);
+Irssi::command('^window log off');
diff --git a/_testing/report-test.zsh b/_testing/report-test.zsh
index 27e6b83..d3217ed 100755
--- a/_testing/report-test.zsh
+++ b/_testing/report-test.zsh
@@ -40,12 +40,23 @@ for scriptfile ($filelist) {
     if [[ $MARKDOWN_REPORT == 1 ]] { print -n '](Test/'$scriptfile:t:r'/)' }
     print -n $T
     local pass=0
+
+    # LOAD
     if [[ -f "Test/${scriptfile:t:r}/failed.yml" ]] { print -n '  '$failmark'   ' } \
-    else { print -n '  '$passmark'   '; ((++pass)) }; print -n $T
+    elif [[ -f "Test/${scriptfile:t:r}/info.yml" ]] { print -n '  '$passmark'   '; ((++pass)) } \
+    else { print -n '  '$skipmark'   '; }; print -n $T
+
+    # HDR
     if { grep -qs 'Severity: 6' "Test/${scriptfile:t:r}/perlcritic.log" } { print -n '  '$failmark'   ' } \
+    elif { grep -qs 'command not found:' "Test/${scriptfile:t:r}/perlcritic.log" } { print -n '  '$skipmark'   ' } \
     else { print -n '  '$passmark'   '; ((++pass)) }; print -n $T
+
+    # CRIT
     if { grep -qs 'Code before strictures are enabled\|Two-argument "open" used' "Test/${scriptfile:t:r}/perlcritic.log" }  { print -n '  '$failmark'   ' } \
+    elif { grep -qs 'command not found:' "Test/${scriptfile:t:r}/perlcritic.log" } { print -n '  '$skipmark'   ' } \
     else { print -n '  '$passmark'   '; ((++pass)) }; print -n $T
+
+    # SCORE
     perl -ne '$score += $1 -1 if /Severity: (\d+)/; END { printf "%3d", $score }' "Test/${scriptfile:t:r}/perlcritic.log" 2>/dev/null
     print -n '   '$T
     if [[ $pass -lt 3 ]]  {
diff --git a/_testing/run-test.zsh b/_testing/run-test.zsh
index ac34c06..5a508f5 100755
--- a/_testing/run-test.zsh
+++ b/_testing/run-test.zsh
@@ -1,5 +1,4 @@
 #!/bin/zsh
-mkdir -p Test
 local base_path="`pwd`"
 local test_script="$base_path/_testing/_irssi_test.pl"
 
@@ -7,8 +6,8 @@ local test_script="$base_path/_testing/_irssi_test.pl"
 
 for scriptfile ($filelist) {
     rm -rf "Test/${scriptfile:t:r}"
-    mkdir "Test/${scriptfile:t:r}"
-    perlcritic --theme certrule --exclude RequireEndWithOne -2 $scriptfile >"Test/${scriptfile:t:r}/perlcritic.log"
+    mkdir -p "Test/${scriptfile:t:r}"
+    perlcritic --theme certrule --exclude RequireEndWithOne -2 $scriptfile >"Test/${scriptfile:t:r}/perlcritic.log" 2>&1
     pushd Test
     rm -fr .home
     mkdir .home