Merge pull request #22 from ecraven/json

Add json output, fix #6

.travis.yml

@@ -22,6 +22,7 @@ matrix:
           - g++-4.8
           - cmake
           - libglib2.0-dev
+          - jq
     # - env: COMPILER_VERSION=3.5
     #   os: linux
     #   compiler: clang++

CMakeLists.txt

@@ -144,7 +144,9 @@ if (BUILD_TESTS)
 
   add_sdcv_shell_test(t_list)
   add_sdcv_shell_test(t_use)
+  add_sdcv_shell_test(t_only_data_dir)
   add_sdcv_shell_test(t_synonyms)
+  add_sdcv_shell_test(t_json)
   add_sdcv_shell_test(t_interactive)
   add_sdcv_shell_test(t_utf8output)
   add_sdcv_shell_test(t_utf8input)

src/libwrapper.cpp

@@ -248,7 +248,7 @@ void Library::LookupData(const std::string &str, TSearchResultList& res_list)
 		}
 }
 
-void Library::print_search_result(FILE *out, const TSearchResult & res)
+void Library::print_search_result(FILE *out, const TSearchResult & res, bool &first_result)
 {
 	std::string loc_bookname, loc_def, loc_exp;
 
@@ -257,18 +257,30 @@ void Library::print_search_result(FILE *out, const TSearchResult & res)
 		loc_def = utf8_to_locale_ign_err(res.def);
 		loc_exp = utf8_to_locale_ign_err(res.exp);
 	}
+        if(json_) {
+          if(!first_result) {
+            fputs(",", out);
+          } else {
+            first_result=false;
+          }
+          fprintf(out,"{\"dict\": \"%s\",\"word\":\"%s\",\"definition\":\"%s\"}",
+                  json_escape_string(res.bookname).c_str(),
+                  json_escape_string(res.def).c_str(),
+                  json_escape_string(res.exp).c_str());
 
-	fprintf(out,
+        } else {
-            "-->%s%s%s\n"
+          fprintf(out,
-            "-->%s%s%s\n"
+                  "-->%s%s%s\n"
-            "%s\n\n",
+                  "-->%s%s%s\n"
-            colorize_output_ ? NAME_OF_DICT_VISFMT : "",
+                  "%s\n\n",
-            utf8_output_ ? res.bookname.c_str() : loc_bookname.c_str(),
+                  colorize_output_ ? NAME_OF_DICT_VISFMT : "",
-            colorize_output_ ? ESC_END : "",
+                  utf8_output_ ? res.bookname.c_str() : loc_bookname.c_str(),
-            colorize_output_ ? SEARCH_TERM_VISFMT : "",
+                  colorize_output_ ? ESC_END : "",
-            utf8_output_ ? res.def.c_str() : loc_def.c_str(),
+                  colorize_output_ ? SEARCH_TERM_VISFMT : "",
-            colorize_output_ ? ESC_END : "",
+                  utf8_output_ ? res.def.c_str() : loc_def.c_str(),
-            utf8_output_ ? res.exp.c_str() : loc_exp.c_str());
+                  colorize_output_ ? ESC_END : "",
+                  utf8_output_ ? res.exp.c_str() : loc_exp.c_str());
+        }
 }
 
 namespace {
@@ -346,6 +358,11 @@ bool Library::process_phrase(const char *loc_str, IReadLine &io, bool force)
 		/*nothing*/;
 	}
 
+	sdcv_pager pager(force);
+	bool first_result = true;
+	if(json_) {
+	  fputc('[', pager.get_stream());
+	}
 	if (!res_list.empty()) {
 		/* try to be more clever, if there are
 		   one or zero results per dictionary show all
@@ -370,8 +387,9 @@ bool Library::process_phrase(const char *loc_str, IReadLine &io, bool force)
 		}//if (!force)
 
 		if (!show_all_results && !force) {
-			printf(_("Found %zu items, similar to %s.\n"), res_list.size(),
+                  if(!json_)
-			       utf8_output_ ? get_impl(str) : utf8_to_locale_ign_err(get_impl(str)).c_str());
+                    printf(_("Found %zu items, similar to %s.\n"), res_list.size(),
+                           utf8_output_ ? get_impl(str) : utf8_to_locale_ign_err(get_impl(str)).c_str());
 			for (size_t i = 0; i < res_list.size(); ++i) {
                 const std::string loc_bookname = utf8_to_locale_ign_err(res_list[i].bookname);
                 const std::string loc_def = utf8_to_locale_ign_err(res_list[i].def);
@@ -390,9 +408,8 @@ bool Library::process_phrase(const char *loc_str, IReadLine &io, bool force)
 				choice_readline->read(_("Your choice[-1 to abort]: "), str_choise);
 				sscanf(str_choise.c_str(), "%d", &choise);
 				if (choise >= 0 && choise < int(res_list.size())) {
-					sdcv_pager pager;
                     io.add_to_history(res_list[choise].def.c_str());
-					print_search_result(pager.get_stream(), res_list[choise]);
+					print_search_result(pager.get_stream(), res_list[choise], first_result);
 					break;
 				} else if (choise == -1){
 					break;
@@ -401,20 +418,24 @@ bool Library::process_phrase(const char *loc_str, IReadLine &io, bool force)
 					       res_list.size()-1);
 			}
 		} else {
-			sdcv_pager pager(force);
+                  for (const TSearchResult& search_res : res_list) {
-			fprintf(pager.get_stream(), _("Found %zu items, similar to %s.\n"),
+                    if(!json_)
-                    res_list.size(), utf8_output_ ? get_impl(str) : utf8_to_locale_ign_err(get_impl(str)).c_str());
+                      fprintf(pager.get_stream(), _("Found %zu items, similar to %s.\n"),
-			for (const TSearchResult& search_res : res_list)
+                              res_list.size(), utf8_output_ ? get_impl(str) : utf8_to_locale_ign_err(get_impl(str)).c_str());
-				print_search_result(pager.get_stream(), search_res);
+                    print_search_result(pager.get_stream(), search_res, first_result);
+                  }
 		}
 
 	} else {
 		std::string loc_str;
 		if (!utf8_output_)
 			loc_str = utf8_to_locale_ign_err(get_impl(str));
-
+                if(!json_)
-		printf(_("Nothing similar to %s, sorry :(\n"), utf8_output_ ? get_impl(str) : loc_str.c_str());
+                  printf(_("Nothing similar to %s, sorry :(\n"), utf8_output_ ? get_impl(str) : loc_str.c_str());
 	}
 
+        if(json_) {
+          fputs("]\n", pager.get_stream());
+        }
 	return true;
 }

src/libwrapper.hpp

@@ -25,19 +25,22 @@ typedef std::vector<TSearchResult> TSearchResultList;
 //of it
 class Library : public Libs {
 public:
-	Library(bool uinput, bool uoutput, bool colorize_output):
+	Library(bool uinput, bool uoutput, bool colorize_output, bool use_json)
-		utf8_input_(uinput), utf8_output_(uoutput), colorize_output_(colorize_output) {}
+	: utf8_input_(uinput), utf8_output_(uoutput), colorize_output_(colorize_output), json_(use_json) {
+          setVerbose(!use_json);
+        }
 
 	bool process_phrase(const char *loc_str, IReadLine &io, bool force = false);
 private:
 	bool utf8_input_;
-    bool utf8_output_;
+	bool utf8_output_;
-    bool colorize_output_;
+	bool colorize_output_;
+	bool json_;
 
 	void SimpleLookup(const std::string &str, TSearchResultList& res_list);
 	void LookupWithFuzzy(const std::string &str, TSearchResultList& res_list);
 	void LookupWithRule(const std::string &str, TSearchResultList& res_lsit);
 	void LookupData(const std::string &str, TSearchResultList& res_list);
-	void print_search_result(FILE *out, const TSearchResult & res);
+	void print_search_result(FILE *out, const TSearchResult & res, bool &first_result);
 };
 

src/sdcv.cpp

@@ -59,7 +59,7 @@ namespace glib
     using StrArr = ResourceWrapper<gchar *, gchar *, free_str_array>;
 }
 
-static void list_dicts(const std::list<std::string> &dicts_dir_list);
+static void list_dicts(const std::list<std::string> &dicts_dir_list, bool use_json);
 
 int main(int argc, char *argv[]) try {
     setlocale(LC_ALL, "");
@@ -75,9 +75,11 @@ int main(int argc, char *argv[]) try {
     gboolean show_list_dicts = FALSE;
     glib::StrArr use_dict_list;
     gboolean non_interactive = FALSE;
+    gboolean json_output = FALSE;
     gboolean utf8_output = FALSE;
     gboolean utf8_input = FALSE;
     glib::CharStr opt_data_dir;
+    gboolean only_data_dir = FALSE;
     gboolean colorize = FALSE;
 
     const GOptionEntry entries[] = {
@@ -90,6 +92,8 @@ int main(int argc, char *argv[]) try {
           _("bookname") },
         { "non-interactive", 'n', 0, G_OPTION_ARG_NONE, &non_interactive,
           _("for use in scripts"), nullptr },
+        { "json-output", 'j', 0, G_OPTION_ARG_NONE, &json_output,
+          _("print the result formatted as JSON."), nullptr },
         { "utf8-output", '0', 0, G_OPTION_ARG_NONE, &utf8_output,
           _("output must be in utf8"), nullptr },
         { "utf8-input", '1', 0, G_OPTION_ARG_NONE, &utf8_input,
@@ -97,6 +101,8 @@ int main(int argc, char *argv[]) try {
         { "data-dir", '2', 0, G_OPTION_ARG_STRING, get_addr(opt_data_dir),
           _("use this directory as path to stardict data directory"),
           _("path/to/dir") },
+        { "only-data-dir", 'x', 0, G_OPTION_ARG_NONE, &only_data_dir,
+          _("only use the dictionaries in data-dir, do not search in user and system directories"), nullptr },
         { "color", 'c', 0, G_OPTION_ARG_NONE, &colorize,
           _("colorize the output"), nullptr },
         {},
@@ -122,10 +128,12 @@ int main(int argc, char *argv[]) try {
     const gchar *stardict_data_dir = g_getenv("STARDICT_DATA_DIR");
     std::string data_dir;
     if (!opt_data_dir) {
+      if (!only_data_dir) {
         if (stardict_data_dir)
             data_dir = stardict_data_dir;
         else
             data_dir = "/usr/share/stardict/dic";
+      }
     } else {
         data_dir = get_impl(opt_data_dir);
     }
@@ -134,13 +142,12 @@ int main(int argc, char *argv[]) try {
     if (!homedir)
         homedir = g_get_home_dir();
 
-    const std::list<std::string> dicts_dir_list = {
+    std::list<std::string> dicts_dir_list;
-        std::string(homedir) + G_DIR_SEPARATOR + ".stardict" + G_DIR_SEPARATOR + "dic",
+    if(!only_data_dir)
-        data_dir
+      dicts_dir_list.push_back(std::string(homedir) + G_DIR_SEPARATOR + ".stardict" + G_DIR_SEPARATOR + "dic");
-    };
+    dicts_dir_list.push_back(data_dir);
-
     if (show_list_dicts) {
-        list_dicts(dicts_dir_list);
+        list_dicts(dicts_dir_list, json_output);
         return EXIT_SUCCESS;
     }
 
@@ -192,7 +199,7 @@ int main(int argc, char *argv[]) try {
         fprintf(stderr, _("g_mkdir failed: %s\n"), strerror(errno));
     }
 
-    Library lib(utf8_input, utf8_output, colorize);
+    Library lib(utf8_input, utf8_output, colorize, json_output);
     lib.load(dicts_dir_list, order_list, disable_list);
 
     std::unique_ptr<IReadLine> io(create_readline_object());
@@ -205,7 +212,7 @@ int main(int argc, char *argv[]) try {
 
         std::string phrase;
         while (io->read(_("Enter word or phrase: "), phrase)) {
-            if (!lib.process_phrase(phrase.c_str(), *io))
+          if (!lib.process_phrase(phrase.c_str(), *io))
                 return EXIT_FAILURE;
             phrase.clear();
         }
@@ -220,17 +227,32 @@ int main(int argc, char *argv[]) try {
     exit(EXIT_FAILURE);
 }
 
-static void list_dicts(const std::list<std::string> &dicts_dir_list)
+static void list_dicts(const std::list<std::string> &dicts_dir_list, bool use_json)
 {
+  bool first_entry = true;
+  if(!use_json)
     printf(_("Dictionary's name   Word count\n"));
-    std::list<std::string> order_list, disable_list;
+  else
-    for_each_file(dicts_dir_list, ".ifo", order_list,
+    fputc('[', stdout);
-                  disable_list, [](const std::string &filename, bool) -> void {
+  std::list<std::string> order_list, disable_list;
-                      DictInfo dict_info;
+  for_each_file(dicts_dir_list, ".ifo", order_list,
-                      if (dict_info.load_from_ifo_file(filename, false)) {
+                disable_list, [use_json, &first_entry](const std::string &filename, bool) -> void {
-                          const std::string bookname = utf8_to_locale_ign_err(dict_info.bookname);
+                  DictInfo dict_info;
-                          printf("%s    %d\n", bookname.c_str(), dict_info.wordcount);
+                  if (dict_info.load_from_ifo_file(filename, false)) {
+                    const std::string bookname = utf8_to_locale_ign_err(dict_info.bookname);
+                    if(use_json) {
+                      if(first_entry) {
+                        first_entry=false;
+                      } else {
+                        fputc(',', stdout); // comma between entries
                       }
-                  });
+                      printf("{\"name\": \"%s\", \"wordcount\": \"%d\"}", json_escape_string(bookname).c_str(), dict_info.wordcount);
+                    } else {
+                      printf("%s    %d\n", bookname.c_str(), dict_info.wordcount);
+                    }
+                  }
+                });
+  if(use_json)
+    fputs("]\n", stdout);
 
 }

src/stardict_lib.cpp

@@ -439,7 +439,7 @@ namespace {
             if (idxfile)
                 fclose(idxfile);
         }
-        bool load(const std::string& url, gulong wc, gulong fsize) override;
+        bool load(const std::string& url, gulong wc, gulong fsize, bool verbose) override;
         const gchar *get_key(glong idx) override;
         void get_data(glong idx) override { get_key(idx); }
         const gchar *get_key_and_data(glong idx) override {
@@ -481,7 +481,7 @@ namespace {
         const gchar *read_first_on_page_key(glong page_idx);
         const gchar *get_first_on_page_key(glong page_idx);
         bool load_cache(const std::string& url);
-        bool save_cache(const std::string& url);
+        bool save_cache(const std::string& url, bool verbose);
         static std::list<std::string> get_cache_variant(const std::string& url);
     };
 
@@ -492,7 +492,7 @@ namespace {
     public:
         WordListIndex() : idxdatabuf(nullptr)	{}
         ~WordListIndex() { g_free(idxdatabuf); }
-        bool load(const std::string& url, gulong wc, gulong fsize) override;
+        bool load(const std::string& url, gulong wc, gulong fsize, bool verbose) override;
         const gchar *get_key(glong idx) override { return wordlist[idx]; }
         void get_data(glong idx) override;
         const gchar *get_key_and_data(glong idx) override {
@@ -592,7 +592,7 @@ namespace {
         return res;
     }
 
-    bool OffsetIndex::save_cache(const std::string& url)
+    bool OffsetIndex::save_cache(const std::string& url, bool verbose)
     {
         const std::list<std::string> vars = get_cache_variant(url);
         for (const std::string&  item : vars) {
@@ -604,13 +604,15 @@ namespace {
             if (fwrite(&wordoffset[0], sizeof(wordoffset[0]), wordoffset.size(), out)!=wordoffset.size())
                 continue;
             fclose(out);
-            printf("save to cache %s\n", url.c_str());
+            if(verbose) {
+              printf("save to cache %s\n", url.c_str());
+            }
             return true;
         }
         return false;
     }
 
-    bool OffsetIndex::load(const std::string& url, gulong wc, gulong fsize)
+    bool OffsetIndex::load(const std::string& url, gulong wc, gulong fsize, bool verbose)
     {
         wordcount=wc;
         gulong npages=(wc-1)/ENTR_PER_PAGE+2;
@@ -633,7 +635,7 @@ namespace {
                 p1 += index_size;
             }
             wordoffset[j]=p1-idxdatabuffer;
-            if (!save_cache(url))
+            if (!save_cache(url, verbose))
                 fprintf(stderr, "cache update failed\n");
         }
 
@@ -741,7 +743,7 @@ namespace {
         return bFound;
     }
 
-    bool WordListIndex::load(const std::string& url, gulong wc, gulong fsize)
+    bool WordListIndex::load(const std::string& url, gulong wc, gulong fsize, bool verbose)
     {
         gzFile in = gzopen(url.c_str(), "rb");
         if (in == nullptr)
@@ -851,7 +853,7 @@ bool Dict::Lookup(const char *str, glong &idx) {
 	return syn_file->lookup(str, idx) || idx_file->lookup(str, idx);
 }
 
-bool Dict::load(const std::string& ifofilename)
+bool Dict::load(const std::string& ifofilename, bool verbose)
 {
 	gulong idxfilesize;
 	if (!load_ifofile(ifofilename, idxfilesize))
@@ -885,7 +887,7 @@ bool Dict::load(const std::string& ifofilename)
 		idx_file.reset(new OffsetIndex);
 	}
 
-	if (!idx_file->load(fullfilename, wordcount, idxfilesize))
+	if (!idx_file->load(fullfilename, wordcount, idxfilesize, verbose))
 		return false;
 
 	fullfilename=ifofilename;
@@ -939,7 +941,7 @@ Libs::~Libs()
 void Libs::load_dict(const std::string& url)
 {
 	Dict *lib=new Dict;
-	if (lib->load(url))
+	if (lib->load(url, verbose_))
 		oLib.push_back(lib);
 	else
 		delete lib;

src/stardict_lib.hpp

@@ -87,7 +87,7 @@ public:
 	guint32 wordentry_size;  
 
 	virtual ~IIndexFile() {}
-	virtual bool load(const std::string& url, gulong wc, gulong fsize) = 0;
+	virtual bool load(const std::string& url, gulong wc, gulong fsize, bool verbose) = 0;
 	virtual const gchar *get_key(glong idx) = 0;
 	virtual void get_data(glong idx) = 0;
 	virtual const gchar *get_key_and_data(glong idx) = 0;
@@ -105,9 +105,9 @@ private:
 class Dict : public DictBase {
 public:
 	Dict() {}
-    Dict(const Dict&) = delete;
+	Dict(const Dict&) = delete;
-    Dict& operator=(const Dict&) = delete;
+	Dict& operator=(const Dict&) = delete;
-	bool load(const std::string& ifofilename);
+	bool load(const std::string& ifofilename, bool verbose);
 
 	gulong narticles() const { return wordcount; }
 	const std::string& dict_name() const { return bookname; }
@@ -141,12 +141,13 @@ private:
 class Libs {
 public:
 	Libs(std::function<void(void)> f = std::function<void(void)>()) {
-        progress_func = f;
+		progress_func = f;
-        iMaxFuzzyDistance  = MAX_FUZZY_DISTANCE; //need to read from cfg.
+		iMaxFuzzyDistance  = MAX_FUZZY_DISTANCE; //need to read from cfg.
-    }
+	}
+	void setVerbose(bool verbose) { verbose_ = verbose; }
 	~Libs();
-    Libs(const Libs&) = delete;
+	Libs(const Libs&) = delete;
-    Libs& operator=(const Libs&) = delete;
+	Libs& operator=(const Libs&) = delete;
 
 	void load_dict(const std::string& url);
 	void load(const std::list<std::string>& dicts_dirs, 
@@ -180,7 +181,8 @@ public:
 private:
 	std::vector<Dict *> oLib; // word Libs.
 	int iMaxFuzzyDistance;
-    std::function<void(void)> progress_func;
+	std::function<void(void)> progress_func;
+	bool verbose_;
 };
 
 

src/utils.cpp

@@ -27,6 +27,8 @@
 #include <cstdlib>
 #include <cstdio>
 #include <algorithm>
+#include <sstream>
+#include <iomanip>
 
 #include "utils.hpp"
 
@@ -90,3 +92,27 @@ void for_each_file(const std::list<std::string>& dirs_list, const std::string& s
 	for (const std::string& item : dirs_list)
 		__for_each_file(item, suff, order_list, disable_list, f);			
 }
+
+// based on https://stackoverflow.com/questions/7724448/simple-json-string-escape-for-c/33799784#33799784
+std::string json_escape_string(const std::string &s) {
+  std::ostringstream o;
+  for (auto c = s.cbegin(); c != s.cend(); c++) {
+    switch (*c) {
+    case '"': o << "\\\""; break;
+    case '\\': o << "\\\\"; break;
+    case '\b': o << "\\b"; break;
+    case '\f': o << "\\f"; break;
+    case '\n': o << "\\n"; break;
+    case '\r': o << "\\r"; break;
+    case '\t': o << "\\t"; break;
+    default:
+      if ('\x00' <= *c && *c <= '\x1f') {
+        o << "\\u"
+          << std::hex << std::setw(4) << std::setfill('0') << (int)*c;
+      } else {
+        o << *c;
+      }
+    }
+  }
+  return o.str();
+}

src/utils.hpp

@@ -60,3 +60,4 @@ extern std::string utf8_to_locale_ign_err(const std::string& utf8_str);
 extern void for_each_file(const std::list<std::string>& dirs_list, const std::string& suff,
                           const std::list<std::string>& order_list, const std::list<std::string>& disable_list, 
                           const std::function<void (const std::string&, bool)>& f);
+extern std::string json_escape_string(const std::string &str);

tests/t_json

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -e
+
+SDCV="$1"
+TEST_DIR="$2"
+
+unset SDCV_PAGER
+unset STARDICT_DATA_DIR
+
+test_json() {
+    PARAMS="$1"
+    EXPECTED=$(echo "$2" | jq 'sort')
+    RESULT=$($SDCV $PARAMS | jq 'sort')
+    if [ "$EXPECTED" != "$RESULT"]; then
+        echo "expected $EXPECTED but got $RESULT"
+        exit 1
+    fi
+}
+
+test_json "-x -j -l -n --data-dir \"$TEST_DIR\"" "[{\"name\": \"Test synonyms\", \"wordcount\": \"1\"},{\"name\": \"Sample 1 test dictionary\", \"wordcount\": \"1\"},{\"name\": \"test_dict\", \"wordcount\": \"1\"}]"
+test_json "-x -j -n --data-dir \"$TEST_DIR\" foo" "[{\"dict\": \"Test synonyms\",\"word\":\"test\",\"definition\":\"\nresult of test\"}]"
+test_json "-x -j -n --data-dir \"$TEST_DIR\" foobarbaaz" "[]"
+
+exit 0

tests/t_only_data_dir

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+SDCV="$1"
+TEST_DIR="$2"
+
+unset SDCV_PAGER
+unset STARDICT_DATA_DIR
+
+DICTS=$($SDCV -x -n -l --data-dir "$TEST_DIR" | tail -n +2 | wc -l)
+# the expected result:
+ACTUAL_DICTS=$(find "$TEST_DIR" -name "*.ifo" | wc -l)
+if [ $DICTS -ne $ACTUAL_DICTS ]; then
+   echo "number of dictionaries in $TEST_DIR should be $ACTUAL_DICTS but was $DICTS according to sdcv"
+   exit 1
+fi
+
+exit 0