t_json: add data about new dictionary

This reverts commit 452a4e07fb.

.clang-format

@@ -15,7 +15,7 @@ BreakBeforeBinaryOperators: true
 BreakBeforeTernaryOperators: true
 BreakConstructorInitializersBeforeComma: true
 BinPackParameters: true
-ColumnLimit:     0
+ColumnLimit:     120
 ConstructorInitializerAllOnOneLineOrOnePerLine: false
 DerivePointerAlignment: false
 ExperimentalAutoDetectBinPacking: false

.github/workflows/main.yml

@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
-      - uses: jwlawson/actions-setup-cmake@v1.0
+      - uses: jwlawson/actions-setup-cmake@v1.4
         with:
           cmake-version: '3.5.1'
           github-api-token: ${{ secrets.GITHUB_TOKEN }}

CMakeLists.txt

@@ -3,6 +3,10 @@ project(sdcv)
 cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
 cmake_policy(VERSION 3.5)
 
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+set(CMAKE_CXX_EXTENSIONS False)
+
 include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/compiler.cmake")
 
 set(ZLIB_FIND_REQUIRED True)
@@ -91,7 +95,7 @@ set(CPACK_PACKAGE_VENDOR "Evgeniy Dushistov <dushistov@mail.ru>")
 set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.org")
 set(CPACK_PACKAGE_VERSION_MAJOR "0")
 set(CPACK_PACKAGE_VERSION_MINOR "5")
-set(CPACK_PACKAGE_VERSION_PATCH "3")
+set(CPACK_PACKAGE_VERSION_PATCH "4")
 
 set(sdcv_VERSION
 	"${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}")
@@ -143,5 +147,7 @@ if (BUILD_TESTS)
   add_sdcv_shell_test(t_utf8input)
   add_sdcv_shell_test(t_datadir)
   add_sdcv_shell_test(t_return_code)
+  add_sdcv_shell_test(t_multiple_results)
+  add_sdcv_shell_test(t_newlines_in_ifo)
 
 endif (BUILD_TESTS)

cmake/compiler.cmake

@@ -16,19 +16,6 @@ if (NOT DEFINED SDCV_COMPILER_IS_GCC_COMPATIBLE)
   endif()
 endif()
 
-if (MSVC AND (MSVC_VERSION LESS 1900))
-  message(FATAL_ERROR "MSVC version ${MSVC_VERSION} have no full c++11 support")
-elseif (MSVC)
-  add_definitions(-DNOMINMAX)
-elseif (NOT MSVC)
-  check_cxx_compiler_flag("-std=c++11" CXX_SUPPORTS_CXX11)
-  if (CXX_SUPPORTS_CXX11)
-    append("-std=c++11" CMAKE_CXX_FLAGS)
-  else ()
-    message(FATAL_ERROR "sdcv requires C++11 support but the '-std=c++11' flag isn't supported.")
-  endif()
-endif ()
-
 if (SDCV_COMPILER_IS_GCC_COMPATIBLE)
   append("-Wall" "-Wextra" "-Wformat-security" "-Wcast-align" "-Werror=format" "-Wcast-qual" CMAKE_C_FLAGS)
   append("-Wall" "-pedantic" "-Wextra" "-Wformat-security" "-Wcast-align" "-Werror=format" "-Wcast-qual" CMAKE_CXX_FLAGS)

src/libwrapper.cpp

@@ -199,14 +199,18 @@ static std::string parse_data(const gchar *data, bool colorize_output)
 
 void Library::SimpleLookup(const std::string &str, TSearchResultList &res_list)
 {
-    glong ind;
+    std::set<glong> wordIdxs;
     res_list.reserve(ndicts());
-    for (gint idict = 0; idict < ndicts(); ++idict)
-        if (SimpleLookupWord(str.c_str(), ind, idict))
-            res_list.push_back(
-                TSearchResult(dict_name(idict),
-                              poGetWord(ind, idict),
-                              parse_data(poGetWordData(ind, idict), colorize_output_)));
+    for (gint idict = 0; idict < ndicts(); ++idict) {
+        wordIdxs.clear();
+        if (SimpleLookupWord(str.c_str(), wordIdxs, idict))
+            for (auto &wordIdx : wordIdxs)
+                res_list.push_back(
+                    TSearchResult(dict_name(idict),
+                                  poGetWord(wordIdx, idict),
+                                  parse_data(poGetWordData(wordIdx, idict),
+                                             colorize_output_)));
+    }
 }
 
 void Library::LookupWithFuzzy(const std::string &str, TSearchResultList &res_list)

src/mapfile.hpp

@@ -7,6 +7,7 @@
 #ifdef HAVE_MMAP
 #include <fcntl.h>
 #include <sys/mman.h>
+#include <sys/stat.h>
 #include <sys/types.h>
 #endif
 #ifdef _WIN32
@@ -40,20 +41,25 @@ inline bool MapFile::open(const char *file_name, unsigned long file_size)
     size = file_size;
 #ifdef HAVE_MMAP
     if ((mmap_fd = ::open(file_name, O_RDONLY)) < 0) {
-        //g_print("Open file %s failed!\n",fullfilename);
+        // g_print("Open file %s failed!\n",fullfilename);
         return false;
     }
+    struct stat st;
+    if (fstat(mmap_fd, &st) == -1 || st.st_size < 0 || (st.st_size == 0 && S_ISREG(st.st_mode))
+        || sizeof(st.st_size) > sizeof(file_size) || static_cast<unsigned long>(st.st_size) != file_size) {
+        close(mmap_fd);
+        return false;
+    }
+
     data = (gchar *)mmap(nullptr, file_size, PROT_READ, MAP_SHARED, mmap_fd, 0);
     if ((void *)data == (void *)(-1)) {
-        //g_print("mmap file %s failed!\n",idxfilename);
+        // g_print("mmap file %s failed!\n",idxfilename);
         data = nullptr;
         return false;
     }
 #elif defined(_WIN32)
-    hFile = CreateFile(file_name, GENERIC_READ, 0, nullptr, OPEN_ALWAYS,
-                       FILE_ATTRIBUTE_NORMAL, 0);
-    hFileMap = CreateFileMapping(hFile, nullptr, PAGE_READONLY, 0,
-                                 file_size, nullptr);
+    hFile = CreateFile(file_name, GENERIC_READ, 0, nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
+    hFileMap = CreateFileMapping(hFile, nullptr, PAGE_READONLY, 0, file_size, nullptr);
     data = (gchar *)MapViewOfFile(hFileMap, FILE_MAP_READ, 0, 0, file_size);
 #else
     gsize read_len;

src/sdcv.cpp

@@ -83,6 +83,7 @@ try {
     glib::CharStr opt_data_dir;
     gboolean only_data_dir = FALSE;
     gboolean colorize = FALSE;
+    glib::StrArr word_list;
 
     const GOptionEntry entries[] = {
         { "version", 'v', 0, G_OPTION_ARG_NONE, &show_version,
@@ -96,6 +97,8 @@ try {
           _("for use in scripts"), nullptr },
         { "json-output", 'j', 0, G_OPTION_ARG_NONE, &json_output,
           _("print the result formatted as JSON"), nullptr },
+        { "json", 'j', 0, G_OPTION_ARG_NONE, &json_output,
+          _("print the result formatted as JSON"), nullptr },
         { "exact-search", 'e', 0, G_OPTION_ARG_NONE, &no_fuzzy,
           _("do not fuzzy-search for similar words, only return exact matches"), nullptr },
         { "utf8-output", '0', 0, G_OPTION_ARG_NONE, &utf8_output,
@@ -109,11 +112,13 @@ try {
           _("only use the dictionaries in data-dir, do not search in user and system directories"), nullptr },
         { "color", 'c', 0, G_OPTION_ARG_NONE, &colorize,
           _("colorize the output"), nullptr },
+        { G_OPTION_REMAINING, 0, 0, G_OPTION_ARG_FILENAME_ARRAY, get_addr(word_list),
+          _("search terms"), _(" words") },
         {},
     };
 
     glib::Error error;
-    GOptionContext *context = g_option_context_new(_(" words"));
+    GOptionContext *context = g_option_context_new(nullptr);
     g_option_context_set_help_enabled(context, TRUE);
     g_option_context_add_main_entries(context, entries, nullptr);
     const gboolean parse_res = g_option_context_parse(context, &argc, &argv, get_addr(error));
@@ -210,14 +215,19 @@ try {
     lib.load(dicts_dir_list, order_list, disable_list);
 
     std::unique_ptr<IReadLine> io(create_readline_object());
-    if (optind < argc) {
+    if (word_list != nullptr) {
         search_result rval = SEARCH_SUCCESS;
-        for (int i = optind; i < argc; ++i)
-            if ((rval = lib.process_phrase(argv[i], *io, non_interactive)) != SEARCH_SUCCESS) {
-                return rval;
-            }
+        gchar **p = get_impl(word_list);
+        while (*p) {
+            search_result this_rval = lib.process_phrase(*p++, *io, non_interactive);
+            // If we encounter any error, save it but continue through the word
+            // list to check all requested words.
+            if (rval == SEARCH_SUCCESS)
+                rval = this_rval;
+        }
+        if (rval != SEARCH_SUCCESS)
+            return rval;
     } else if (!non_interactive) {
-
         std::string phrase;
         while (io->read(_("Enter word or phrase: "), phrase)) {
             if (lib.process_phrase(phrase.c_str(), *io) == SEARCH_FAILURE)

src/stardict_lib.hpp

@@ -1,11 +1,10 @@
 #pragma once
 
-#include <cstdio>
 #include <cstring>
 #include <functional>
 #include <list>
-#include <map>
 #include <memory>
+#include <set>
 #include <string>
 #include <vector>
 
@@ -29,7 +28,7 @@ inline void set_uint32(gchar *addr, guint32 val)
 struct cacheItem {
     guint32 offset;
     gchar *data;
-    //write code here to make it inline
+    // write code here to make it inline
     cacheItem() { data = nullptr; }
     ~cacheItem() { g_free(data); }
 };
@@ -67,7 +66,7 @@ private:
     gint cache_cur = 0;
 };
 
-//this structure contain all information about dictionary
+// this structure contain all information about dictionary
 struct DictInfo {
     std::string ifo_file_name;
     guint32 wordcount;
@@ -96,17 +95,27 @@ public:
     virtual const gchar *get_key(glong idx) = 0;
     virtual void get_data(glong idx) = 0;
     virtual const gchar *get_key_and_data(glong idx) = 0;
-    virtual bool lookup(const char *str, glong &idx) = 0;
+    virtual bool lookup(const char *str, std::set<glong> &idxs, glong &next_idx) = 0;
+    virtual bool lookup(const char *str, std::set<glong> &idxs)
+    {
+        glong unused_next_idx;
+        return lookup(str, idxs, unused_next_idx);
+    };
 };
 
 class SynFile
 {
 public:
+    SynFile() {}
+    ~SynFile() {}
     bool load(const std::string &url, gulong wc);
-    bool lookup(const char *str, glong &idx);
+    bool lookup(const char *str, std::set<glong> &idxs, glong &next_idx);
+    bool lookup(const char *str, std::set<glong> &idxs);
+    const gchar *get_key(glong idx) { return synlist[idx]; }
 
 private:
-    std::map<std::string, gulong> synonyms;
+    MapFile synfile;
+    std::vector<gchar *> synlist;
 };
 
 class Dict : public DictBase
@@ -133,7 +142,12 @@ public:
         *offset = idx_file->wordentry_offset;
         *size = idx_file->wordentry_size;
     }
-    bool Lookup(const char *str, glong &idx);
+    bool Lookup(const char *str, std::set<glong> &idxs, glong &next_idx);
+    bool Lookup(const char *str, std::set<glong> &idxs)
+    {
+        glong unused_next_idx;
+        return Lookup(str, idxs, unused_next_idx);
+    }
 
     bool LookupWithRule(GPatternSpec *pspec, glong *aIndex, int iBuffLen);
 
@@ -155,7 +169,7 @@ public:
     Libs(std::function<void(void)> f = std::function<void(void)>())
     {
         progress_func = f;
-        iMaxFuzzyDistance = MAX_FUZZY_DISTANCE; //need to read from cfg.
+        iMaxFuzzyDistance = MAX_FUZZY_DISTANCE; // need to read from cfg.
     }
     void setVerbose(bool verbose) { verbose_ = verbose; }
     void setFuzzy(bool fuzzy) { fuzzy_ = fuzzy; }
@@ -181,15 +195,12 @@ public:
             return nullptr;
         return oLib[iLib]->get_data(iIndex);
     }
-    const gchar *poGetCurrentWord(glong *iCurrent);
-    const gchar *poGetNextWord(const gchar *word, glong *iCurrent);
-    const gchar *poGetPreWord(glong *iCurrent);
-    bool LookupWord(const gchar *sWord, glong &iWordIndex, int iLib)
+    bool LookupWord(const gchar *sWord, std::set<glong> &iWordIndices, int iLib)
     {
-        return oLib[iLib]->Lookup(sWord, iWordIndex);
+        return oLib[iLib]->Lookup(sWord, iWordIndices);
     }
-    bool LookupSimilarWord(const gchar *sWord, glong &iWordIndex, int iLib);
-    bool SimpleLookupWord(const gchar *sWord, glong &iWordIndex, int iLib);
+    bool LookupSimilarWord(const gchar *sWord, std::set<glong> &iWordIndices, int iLib);
+    bool SimpleLookupWord(const gchar *sWord, std::set<glong> &iWordIndices, int iLib);
 
     bool LookupWithFuzzy(const gchar *sWord, gchar *reslist[], gint reslist_size);
     gint LookupWithRule(const gchar *sWord, gchar *reslist[]);

tests/not-unix-newlines-ifo/russian/russian.ifo

@@ -0,0 +1,9 @@
+StarDict's dict ifo file
+version=3.0.0
+bookname=Russian-English Dictionary (ru-en)
+wordcount=415144
+idxfilesize=12344255
+sametypesequence=h
+synwordcount=1277580
+author=Vuizur
+description=

tests/stardict-test_multiple_results-2.4.2/test.ifo

@@ -0,0 +1,7 @@
+StarDict's dict ifo file
+version=3.0.0
+bookname=Test multiple results
+wordcount=246
+idxfilesize=5977
+synwordcount=124
+description=

tests/t_json

@@ -18,8 +18,15 @@ test_json() {
     fi
 }
 
-test_json '[{"name": "Test synonyms", "wordcount": "2"},{"name": "Sample 1 test dictionary", "wordcount": "1"},{"name": "test_dict", "wordcount": "1"}]' -x -j -l -n --data-dir "$TEST_DIR"
+test_json '[{"name": "Russian-English Dictionary (ru-en)", "wordcount": "415144"},
+            {"name": "Test synonyms", "wordcount": "2"},
+            {"name": "Test multiple results", "wordcount": "246"},
+            {"name": "Sample 1 test dictionary", "wordcount": "1"},
+            {"name": "test_dict", "wordcount": "1"}]' -x -j -l -n --data-dir "$TEST_DIR"
 test_json '[{"dict": "Test synonyms","word":"test","definition":"\u000aresult of test"}]' -x -j -n --data-dir "$TEST_DIR" foo
 test_json '[]' -x -j -n --data-dir "$TEST_DIR" foobarbaaz
 
+# Test multiple searches, with the first failing.
+test_json '[][{"dict": "Test synonyms","word":"test","definition":"\u000aresult of test"}]' -x -j -n --data-dir "$TEST_DIR" foobarbaaz foo
+
 exit 0

tests/t_multiple_results

@@ -0,0 +1,67 @@
+#!/bin/sh
+
+set -e
+
+SDCV="$1"
+TEST_DIR="$2"
+
+unset SDCV_PAGER
+unset STARDICT_DATA_DIR
+
+test_json() {
+    word="$1"
+    jq_cmp="$2"
+    result="$("$SDCV" --data-dir "$TEST_DIR" -exjn "$word" | sed 's|\\n|\\u000a|g')"
+    cmp_result="$(echo "$result" | jq "$jq_cmp")"
+    if [ "$cmp_result" != "true" ]; then
+        echo "expected '$jq_cmp' to return true, but $result didn't"
+        exit 1
+    fi
+}
+
+# Basic two-result search for the same headword.
+test_json bark \
+    '. == [
+        {"dict":"Test multiple results","word":"bark","definition":"\u000aThe harsh sound made by a dog."},
+        {"dict":"Test multiple results","word":"bark","definition":"\u000aThe tough outer covering of trees and other woody plants."}
+    ]'
+
+# Multi-result search where one word exists as both a synyonym and a separate
+# headword. This ensures that if there is a matching synyonym we don't skip the
+# regular search.
+test_json cat \
+    '. == [
+        {"dict":"Test multiple results","word":"cat","definition":"\u000aA cute animal which (rarely) barks."},
+        {"dict":"Test multiple results","word":"lion","definition":"\u000aA larger cat which might bite your head off."},
+        {"dict":"Test multiple results","word":"panther","definition":"\u000aI know very little about panthers, sorry."}
+    ]'
+
+# Many-result search for a word that matches 120 distinct headwords.
+test_json many_headwords 'length == 120'
+test_json many_headwords 'all(.word == "many_headwords")'
+test_json many_headwords \
+    'to_entries | map(.value.definition == "\u000aDefinition for [many_headwords] entry #\(.key+1) (same headword).") | all'
+
+# Many-result search for 120 words that have the same synonym.
+test_json many_synonyms 'length == 120'
+test_json many_synonyms \
+    'to_entries | map(.value.word == "many_synonyms-\(.key+101)") | all'
+test_json many_synonyms \
+    'to_entries | map(.value.definition == "\u000aDefinition for [many_synonyms-\(.key+101)] (same synonym).") | all'
+
+# Ensure that we don't return more than one result even if a word can be
+# resolved in more than one way.
+#
+# Most well-formed dictionaries don't have entries like this (it basically
+# requires you to have a dictionary where there is a synonym that is identical
+# to a word's headword or multiple identical synyonym entries).
+#
+# This entry was created by creating extra synonyms with different names then
+# modifying the .syn file manually.
+test_json many_resolution_paths \
+    '. == [
+        {"dict":"Test multiple results","word":"many_resolution_paths",
+         "definition":"\u000aDefinition for [many_resolution_paths] headword (same word, multiple synonym entries)."}
+    ]'
+
+exit 0

tests/t_newlines_in_ifo

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+PATH_TO_SDCV="$1"
+TEST_DIR="$2"
+
+unset SDCV_PAGER
+unset STARDICT_DATA_DIR
+
+RES=$("$PATH_TO_SDCV" -n -x --data-dir="$TEST_DIR/not-unix-newlines-ifo" -l | tail -n 1)
+
+if [ "$RES" = "Russian-English Dictionary (ru-en)    415144" ]; then
+  exit 0
+else
+  echo "test failed, unexpected result: $RES" >&2
+  exit 1
+fi