From: Adam Dullage <redacted>
Date: Thu, 15 Sep 2022 12:20:26 +0000 (+0100)
Subject: Searching improvements
X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=71217cebfec1ae769a2e6a013ae2cff8e9e26ad0;p=flatnotes.git

Searching improvements
---

diff --git a/flatnotes/flatnotes.py b/flatnotes/flatnotes.py
index edb688b..d514f1a 100644
--- a/flatnotes/flatnotes.py
+++ b/flatnotes/flatnotes.py
@@ -3,15 +3,16 @@ import logging
 import os
 import re
 from datetime import datetime
-from typing import List, Set, Tuple
+from typing import List, Literal, Set, Tuple
 
 import whoosh
 from helpers import empty_dir, re_extract, strip_ext
 from whoosh import writing
-from whoosh.analysis import CharsetFilter, KeywordAnalyzer, StemmingAnalyzer
-from whoosh.fields import ID, STORED, TEXT, SchemaClass
+from whoosh.analysis import CharsetFilter, StemmingAnalyzer
+from whoosh.fields import DATETIME, ID, KEYWORD, TEXT, SchemaClass
 from whoosh.index import Index
 from whoosh.qparser import MultifieldParser
+from whoosh.qparser.dateparse import DateParserPlugin
 from whoosh.query import Every
 from whoosh.searching import Hit
 from whoosh.support.charset import accent_map
@@ -24,10 +25,12 @@ StemmingFoldingAnalyzer = StemmingAnalyzer() | CharsetFilter(accent_map)
 
 class IndexSchema(SchemaClass):
     filename = ID(unique=True, stored=True)
-    last_modified = STORED()
-    title = TEXT(field_boost=2, analyzer=StemmingFoldingAnalyzer)
+    last_modified = DATETIME(stored=True, sortable=True)
+    title = TEXT(
+        field_boost=2, analyzer=StemmingFoldingAnalyzer, sortable=True
+    )
     content = TEXT(analyzer=StemmingFoldingAnalyzer)
-    tags = TEXT(analyzer=KeywordAnalyzer())
+    tags = KEYWORD(lowercase=True)
 
 
 class InvalidTitleError(Exception):
@@ -105,7 +108,10 @@ class SearchResult(Note):
         super().__init__(flatnotes, strip_ext(hit["filename"]))
 
         self._matched_fields = self._get_matched_fields(hit.matched_terms())
-        self._rank = hit.rank
+        # If the search was ordered using a text field then hit.score is the
+        # value of that field. This isn't useful so only set self._score if it
+        # is a float.
+        self._score = hit.score if type(hit.score) is float else None
         self._title_highlights = (
             hit.highlights("title", text=self.title)
             if "title" in self._matched_fields
@@ -126,8 +132,8 @@ class SearchResult(Note):
         )
 
     @property
-    def rank(self):
-        return self._rank
+    def score(self):
+        return self._score
 
     @property
     def title_highlights(self):
@@ -211,7 +217,7 @@ class Flatnotes(object):
         tag_string = " ".join(tag_set)
         writer.update_document(
             filename=note.filename,
-            last_modified=note.last_modified,
+            last_modified=datetime.fromtimestamp(note.last_modified),
             title=note.title,
             content=content_ex_tags,
             tags=tag_string,
@@ -244,7 +250,8 @@ class Flatnotes(object):
                     logging.info(f"'{idx_filename}' removed from index")
                 # Update modified
                 elif (
-                    os.path.getmtime(idx_filepath) != idx_note["last_modified"]
+                    datetime.fromtimestamp(os.path.getmtime(idx_filepath))
+                    != idx_note["last_modified"]
                 ):
                     logging.info(f"'{idx_filename}' updated")
                     self._add_note_to_index(
@@ -278,15 +285,46 @@ class Flatnotes(object):
             tags = reader.field_terms("tags")
             return [tag for tag in tags]
 
-    def search(self, term: str) -> Tuple[SearchResult, ...]:
+    def search(
+        self,
+        term: str,
+        sort: Literal["score", "title", "last_modified"] = "score",
+        order: Literal["asc", "desc"] = "desc",
+        limit: int = None,
+    ) -> Tuple[SearchResult, ...]:
         """Search the index for the given term."""
         self.update_index_debounced()
+        term = term.strip()
         with self.index.searcher() as searcher:
+            # Parse Query
             if term == "*":
                 query = Every()
             else:
-                query = MultifieldParser(
+                parser = MultifieldParser(
                     ["title", "content", "tags"], self.index.schema
-                ).parse(term)
-            results = searcher.search(query, limit=None, terms=True)
+                )
+                parser.add_plugin(DateParserPlugin())
+                query = parser.parse(term)
+
+            # Determine Sort By
+            # Note: For the 'sort' option, "score" is converted to None as
+            # that is the default for searches anyway and it's quicker for
+            # Whoosh if you specify None.
+            sort = sort if sort in ["title", "last_modified"] else None
+
+            # Determine Sort Direction
+            # Note: Confusingly, when sorting by 'score', reverse = True means
+            # asc so we have to flip the logic for that case!
+            reverse = order == "desc"
+            if sort is None:
+                reverse = not reverse
+
+            # Run Search
+            results = searcher.search(
+                query,
+                sortedby=sort,
+                reverse=reverse,
+                limit=limit,
+                terms=True,
+            )
             return tuple(SearchResult(self, hit) for hit in results)
diff --git a/flatnotes/main.py b/flatnotes/main.py
index 6cdc185..f362f4f 100644
--- a/flatnotes/main.py
+++ b/flatnotes/main.py
@@ -149,10 +149,21 @@ async def get_tags(_: str = Depends(validate_token)):
 
 
 @app.get("/api/search", response_model=List[SearchResultModel])
-async def search(term: str, _: str = Depends(validate_token)):
-    """Perform a full text search for a note."""
+async def search(
+    term: str,
+    sort: Literal["score", "title", "lastModified"] = "score",
+    order: Literal["asc", "desc"] = "desc",
+    limit: int = None,
+    _: str = Depends(validate_token),
+):
+    """Perform a full text search on all notes."""
+    if sort == "lastModified":
+        sort = "last_modified"
     return [
-        SearchResultModel.dump(note_hit) for note_hit in flatnotes.search(term)
+        SearchResultModel.dump(note_hit)
+        for note_hit in flatnotes.search(
+            term, sort=sort, order=order, limit=limit
+        )
     ]
 
 
diff --git a/flatnotes/models.py b/flatnotes/models.py
index f0d5abf..348a7f1 100644
--- a/flatnotes/models.py
+++ b/flatnotes/models.py
@@ -30,7 +30,7 @@ class NotePatchModel(CamelCaseBaseModel):
 
 
 class SearchResultModel(CamelCaseBaseModel):
-    rank: int
+    score: Optional[float]
     title: str
     last_modified: int
     title_highlights: Optional[str]
@@ -40,7 +40,7 @@ class SearchResultModel(CamelCaseBaseModel):
     @classmethod
     def dump(self, search_result: SearchResult) -> Dict:
         return {
-            "rank": search_result.rank,
+            "score": search_result.score,
             "title": search_result.title,
             "lastModified": search_result.last_modified,
             "titleHighlights": search_result.title_highlights,
diff --git a/flatnotes/src/classes.js b/flatnotes/src/classes.js
index c5913f3..93a10c9 100644
--- a/flatnotes/src/classes.js
+++ b/flatnotes/src/classes.js
@@ -23,7 +23,7 @@ class Note {
 class SearchResult extends Note {
   constructor(searchResult) {
     super(searchResult.title, searchResult.lastModified);
-    this.rank = searchResult.rank;
+    this.score = searchResult.score;
     this.titleHighlights = searchResult.titleHighlights;
     this.contentHighlights = searchResult.contentHighlights;
     this.tagMatches = searchResult.tagMatches;
@@ -32,6 +32,18 @@ class SearchResult extends Note {
   get titleHighlightsOrTitle() {
     return this.titleHighlights ? this.titleHighlights : this.title;
   }
+
+  get includesHighlights() {
+    if (
+      this.titleHighlights ||
+      this.contentHighlights ||
+      (this.tagMatches != null && this.tagMatches.length)
+    ) {
+      return true;
+    } else {
+      return false;
+    }
+  }
 }
 
 export { Note, SearchResult };
diff --git a/flatnotes/src/components/App.vue b/flatnotes/src/components/App.vue
index 82f6c1f..112ca1f 100644
--- a/flatnotes/src/components/App.vue
+++ b/flatnotes/src/components/App.vue
@@ -55,10 +55,6 @@
       v-if="currentView == views.search"
       class="flex-grow-1 search-results-view d-flex flex-column"
     >
-      <SearchInput
-        :initial-value="searchTerm"
-        class="search-input mb-4"
-      ></SearchInput>
       <SearchResults
         :search-term="searchTerm"
         class="flex-grow-1"
diff --git a/flatnotes/src/components/NoteList.vue b/flatnotes/src/components/NoteList.vue
index e9c6825..1a32b40 100644
--- a/flatnotes/src/components/NoteList.vue
+++ b/flatnotes/src/components/NoteList.vue
@@ -45,7 +45,7 @@
   </div>
 </template>
 
-<style lang="scss" >
+<style lang="scss" scoped>
 @import "../colours";
 
 .centered {
@@ -66,7 +66,6 @@
   font-weight: bold;
   font-size: 32px;
   color: $very-muted-text;
-  margin-bottom: 1px solid $very-muted-text;
 }
 
 .note-row {
diff --git a/flatnotes/src/components/SearchResults.vue b/flatnotes/src/components/SearchResults.vue
index e28e031..369f3c2 100644
--- a/flatnotes/src/components/SearchResults.vue
+++ b/flatnotes/src/components/SearchResults.vue
@@ -1,5 +1,8 @@
 <template>
   <div>
+    <!-- Input -->
+    <SearchInput :initial-value="searchTerm" class="mb-1"></SearchInput>
+
     <!-- Searching -->
     <div
       v-if="searchResults == null || searchResults.length == 0"
@@ -14,35 +17,63 @@
 
     <!-- Search Results Loaded -->
     <div v-else>
-      <button type="button" class="bttn mb-3" @click="toggleHighlights">
-        <b-icon :icon="showHighlights ? 'eye-slash' : 'eye'"></b-icon>
-        {{ showHighlights ? "Hide" : "Show" }} Highlights
-      </button>
+      <!-- Controls -->
+      <div class="mb-3">
+        <select v-model="sortBy" class="bttn sort-select">
+          <option
+            v-for="option in sortOptions"
+            :key="option"
+            :value="option"
+            class="p-0"
+          >
+            Order: {{ sortOptionToString(option) }}
+          </option>
+        </select>
+
+        <button
+          v-if="searchResultsIncludeHighlights"
+          type="button"
+          class="bttn"
+          @click="showHighlights = !showHighlights"
+        >
+          <b-icon :icon="showHighlights ? 'eye-slash' : 'eye'"></b-icon>
+          {{ showHighlights ? "Hide" : "Show" }} Highlights
+        </button>
+      </div>
+
+      <!-- Results -->
       <div
-        v-for="result in searchResults"
-        :key="result.title"
-        class="bttn result mb-2"
+        v-for="group in resultsGrouped"
+        :key="group.name"
+        :class="{ 'mb-4': sortByIsGrouped }"
       >
-        <a :href="result.href" @click.prevent="openNote(result.href)">
-          <div class="d-flex align-items-center">
+        <p v-if="sortByIsGrouped" class="group-name">{{ group.name }}</p>
+        <div
+          v-for="result in group.searchResults"
+          :key="result.title"
+          class="bttn result mb-2"
+        >
+          <a :href="result.href" @click.prevent="openNote(result.href)">
+            <div class="d-flex align-items-center">
+              <p
+                class="result-title"
+                v-html="
+                  showHighlights ? result.titleHighlightsOrTitle : result.title
+                "
+              ></p>
+            </div>
             <p
-              class="result-title"
-              v-html="
-                showHighlights ? result.titleHighlightsOrTitle : result.title
-              "
+              v-show="showHighlights"
+              class="result-contents"
+              v-html="result.contentHighlights"
             ></p>
-          </div>
-          <p
-            v-show="showHighlights"
-            class="result-contents"
-            v-html="result.contentHighlights"
-          ></p>
-          <div v-show="showHighlights">
-            <span v-for="tag in result.tagMatches" :key="tag" class="tag mr-2"
-              >#{{ tag }}</span
-            >
-          </div>
-        </a>
+            <div v-show="showHighlights">
+              <span v-for="tag in result.tagMatches" :key="tag" class="tag mr-2"
+                >#{{ tag }}</span
+              >
+            </div>
+          </a>
+        </div>
       </div>
     </div>
   </div>
@@ -51,6 +82,18 @@
 <style lang="scss" scoped>
 @import "../colours";
 
+.sort-select {
+  padding-inline: 6px;
+}
+
+.group-name {
+  padding-left: 8px;
+  font-weight: bold;
+  font-size: 32px;
+  color: $very-muted-text;
+  margin-bottom: 2px;
+}
+
 .result p {
   margin: 0;
 }
@@ -87,12 +130,14 @@ import * as helpers from "../helpers";
 
 import EventBus from "../eventBus";
 import LoadingIndicator from "./LoadingIndicator";
+import SearchInput from "./SearchInput";
 import { SearchResult } from "../classes";
 import api from "../api";
 
 export default {
   components: {
     LoadingIndicator,
+    SearchInput,
   },
 
   props: {
@@ -105,20 +150,51 @@ export default {
       searchFailedMessage: "Failed to load Search Results",
       searchFailedIcon: null,
       searchResults: null,
+      searchResultsIncludeHighlights: null,
+      sortBy: 0,
       showHighlights: true,
     };
   },
 
+  computed: {
+    sortByIsGrouped: function () {
+      return this.sortBy == this.sortOptions.title;
+    },
+
+    resultsGrouped: function () {
+      if (this.sortBy == this.sortOptions.title) {
+        return this.resultsByTitle();
+      } else if (this.sortBy == this.sortOptions.lastModified) {
+        return this.resultsByLastModified();
+      } else {
+        // Default
+        return this.resultsByScore();
+      }
+    },
+  },
+
   watch: {
     searchTerm: function () {
       this.init();
     },
+
+    showHighlights: function () {
+      helpers.setSearchParam(
+        constants.params.showHighlights,
+        this.showHighlights
+      );
+    },
+
+    sortBy: function () {
+      helpers.setSearchParam(constants.params.sortBy, this.sortBy);
+    },
   },
 
   methods: {
     getSearchResults: function () {
       let parent = this;
       this.searchFailed = false;
+      this.searchResultsIncludeHighlights = false;
       api
         .get("/api/search", { params: { term: this.searchTerm } })
         .then(function (response) {
@@ -128,8 +204,15 @@ export default {
             parent.searchFailedMessage = "No Results";
             parent.searchFailed = true;
           } else {
-            response.data.forEach(function (searchResult) {
-              parent.searchResults.push(new SearchResult(searchResult));
+            response.data.forEach(function (responseItem) {
+              let searchResult = new SearchResult(responseItem);
+              parent.searchResults.push(searchResult);
+              if (
+                parent.searchResultsIncludeHighlights == false &&
+                searchResult.includesHighlights
+              ) {
+                parent.searchResultsIncludeHighlights = true;
+              }
             });
           }
         })
@@ -141,16 +224,88 @@ export default {
         });
     },
 
+    resultsByScore: function () {
+      return [
+        {
+          name: "_",
+          searchResults: [...this.searchResults].sort(function (
+            searchResultA,
+            searchResultB
+          ) {
+            return searchResultB.score - searchResultA.score;
+          }),
+        },
+      ];
+    },
+
+    resultsByLastModified: function () {
+      return [
+        {
+          name: "_",
+          searchResults: this.searchResults.sort(function (
+            searchResultA,
+            searchResultB
+          ) {
+            return searchResultB.lastModified - searchResultA.lastModified;
+          }),
+        },
+      ];
+    },
+
+    resultsByTitle: function () {
+      // Set up an empty dictionary of groups
+      let notesGroupedDict = {};
+      let specialCharGroupTitle = "#";
+      [specialCharGroupTitle, ...constants.alphabet].forEach(function (group) {
+        notesGroupedDict[group] = [];
+      });
+
+      // Add results to the group dictionary
+      this.searchResults.forEach(function (searchResult) {
+        let firstCharUpper = searchResult.title[0].toUpperCase();
+        if (constants.alphabet.includes(firstCharUpper)) {
+          notesGroupedDict[firstCharUpper].push(searchResult);
+        } else {
+          notesGroupedDict[specialCharGroupTitle].push(searchResult);
+        }
+      });
+
+      // Convert dict to an array skipping empty groups
+      let notesGroupedArray = [];
+      Object.entries(notesGroupedDict).forEach(function (item) {
+        if (item[1].length) {
+          notesGroupedArray.push({
+            name: item[0],
+            searchResults: item[1].sort(function (
+              SearchResultA,
+              SearchResultB
+            ) {
+              // Sort by title within each group
+              return SearchResultA.title.localeCompare(SearchResultB.title);
+            }),
+          });
+        }
+      });
+
+      // Ensure the array is ordered correctly
+      notesGroupedArray.sort(function (groupA, groupB) {
+        return groupA.name.localeCompare(groupB.name);
+      });
+
+      return notesGroupedArray;
+    },
+
     openNote: function (href) {
       EventBus.$emit("navigate", href);
     },
 
-    toggleHighlights: function () {
-      this.showHighlights = !this.showHighlights;
-      helpers.setSearchParam(
-        constants.params.showHighlights,
-        this.showHighlights
-      );
+    sortOptionToString: function (sortOption) {
+      let sortOptionStrings = {
+        0: "Score",
+        1: "Title",
+        2: "Last Modified",
+      };
+      return sortOptionStrings[sortOption];
     },
 
     init: function () {
@@ -159,16 +314,15 @@ export default {
   },
 
   created: function () {
+    this.sortOptions = constants.searchSortOptions;
     this.init();
 
-    let showHighlightsParam = helpers.getSearchParam(
-      constants.params.showHighlights
+    this.showHighlights = helpers.getSearchParamBool(
+      constants.params.showHighlights,
+      true
     );
-    if (typeof showHighlightsParam == "string") {
-      this.showHighlights = showHighlightsParam === "true";
-    } else {
-      this.showHighlights = true;
-    }
+
+    this.sortBy = helpers.getSearchParamInt(constants.params.sortBy, 0);
   },
 };
 </script>
diff --git a/flatnotes/src/constants.js b/flatnotes/src/constants.js
index 858eafc..d559468 100644
--- a/flatnotes/src/constants.js
+++ b/flatnotes/src/constants.js
@@ -13,6 +13,7 @@ export const params = {
   searchTerm: "term",
   redirect: "redirect",
   showHighlights: "showHighlights",
+  sortBy: "sortBy",
 };
 
 // Other
@@ -44,3 +45,5 @@ export const alphabet = [
   "Y",
   "Z",
 ];
+
+export const searchSortOptions = { score: 0, title: 1, lastModified: 2 };
diff --git a/flatnotes/src/global.scss b/flatnotes/src/global.scss
index bd6da32..c61d84a 100644
--- a/flatnotes/src/global.scss
+++ b/flatnotes/src/global.scss
@@ -44,4 +44,5 @@ a {
 }
 .bttn:hover {
   background-color: $button-background;
+  cursor: pointer;
 }
diff --git a/flatnotes/src/helpers.js b/flatnotes/src/helpers.js
index cc0763a..5e45d98 100644
--- a/flatnotes/src/helpers.js
+++ b/flatnotes/src/helpers.js
@@ -1,6 +1,39 @@
-export function getSearchParam(paramName) {
+export function getSearchParam(paramName, defaultValue = null) {
   let urlSearchParams = new URLSearchParams(window.location.search);
-  return urlSearchParams.get(paramName);
+  let paramValue = urlSearchParams.get(paramName);
+  if (paramValue != null) {
+    return paramValue;
+  } else {
+    return defaultValue;
+  }
+}
+
+export function getSearchParamBool(paramName, defaultValue = null) {
+  let paramValue = getSearchParam(paramName)
+  if (paramValue == null) {
+    return defaultValue
+  }
+  let paramValueLowerCase = paramValue.toLowerCase();
+  if (paramValueLowerCase == "true") {
+    return true;
+  } else if (paramValueLowerCase == "false") {
+    return false;
+  } else {
+    return defaultValue;
+  }
+}
+
+export function getSearchParamInt(paramName, defaultValue = null) {
+  let paramValue = getSearchParam(paramName)
+  if (paramValue == null) {
+    return defaultValue
+  }
+  let paramValueInt = parseInt(paramValue);
+  if (!isNaN(paramValueInt)) {
+    return paramValueInt;
+  } else {
+    return defaultValue;
+  }
 }
 
 export function setSearchParam(paramName, value) {