initial commit

Signed-off-by: ngn <ngn@ngn.tf>

extract/dictionary.go

@@ -0,0 +1,31 @@
+package extract
+
+import "net/url"
+
+type dictionary struct {
+  ShortName string
+  Name string
+  Url string
+}
+
+var Dicts []dictionary = []dictionary{
+  {ShortName: "EN-TR", Name: "Turkish - English", Url: "/turkish-english"},
+  {ShortName: "EN-DE", Name: "German - English", Url: "/german-english"},
+  {ShortName: "EN-ES", Name: "Spanish - English", Url: "/spanish-english"},
+  {ShortName: "EN-FR", Name: "French - English", Url: "/french-english"},
+}
+
+func find_dict(sn string) (*dictionary) {
+  for i := range Dicts {
+    if Dicts[i].ShortName == sn {
+      return &Dicts[i]
+    }
+  }
+
+  return nil
+}
+
+func (d *dictionary) Path(term string) string{
+  term_escaped := url.PathEscape(term)
+  return d.Url + "/" + term_escaped
+}

extract/extractor.go

@@ -0,0 +1,87 @@
+package extract
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+const (
+  BASE_URL = "https://tureng.com/en/"
+  USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3"
+  REFERER = "https://tureng.com/en/turkish-english"
+)
+
+type Extractor struct {
+  client *http.Client
+}
+
+func (e *Extractor) get(path string) (*http.Response, error) {
+  var (
+    err error
+    full_url string
+
+    req *http.Request
+    res *http.Response
+  )
+
+  if full_url, err = url.JoinPath(BASE_URL, path); err != nil {
+    return nil, err
+  }
+
+  if req, err = http.NewRequest("GET", full_url, nil); err != nil {
+    return nil, err
+  }
+
+  // headers
+  req.Header.Set("User-Agent", USER_AGENT)
+  req.Header.Set("Referer", REFERER)
+
+  if res, err = e.client.Do(req); err != nil {
+    return nil, err
+  }
+
+  if res.StatusCode != 200 {
+    return nil, fmt.Errorf("bad response code: %d", res.StatusCode)
+  }
+
+  return res, err
+}
+
+func (e *Extractor) Translate(dict_name string, term string) (*Translation, error) {
+  var (
+    dict *dictionary
+    res *http.Response
+    doc *goquery.Document
+    err error
+  )
+
+  if dict = find_dict(dict_name); dict == nil {
+    return nil, nil
+  }
+
+  term_path := dict.Path(term)
+
+  if res, err = e.get(term_path); err != nil {
+    return nil, fmt.Errorf("failed to get %s: %s", term_path, err.Error())
+  }
+
+  defer res.Body.Close()
+
+  if doc, err = goquery.NewDocumentFromReader(res.Body); err != nil {
+    return nil, err
+  }
+
+  trans := Translation{}
+  err = trans.GetResults(doc)
+
+  return &trans, err
+}
+
+func New() (*Extractor, error) {
+  var extractor Extractor
+  extractor.client = &http.Client{}
+  return &extractor, nil
+}

extract/translation.go

@@ -0,0 +1,113 @@
+package extract
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Entry struct {
+  Text string
+  Detail string
+}
+
+type Translation struct {
+  Suggestions []string
+  Fields []string
+  Entries [][]Entry
+}
+
+func (t *Translation) getSuggestions(doc *goquery.Document) error {
+  doc.Find(".suggestion-list li").Each(func(_ int, sel *goquery.Selection){
+    if link := sel.Find("a"); link != nil {
+      t.Suggestions = append(t.Suggestions, link.Text())
+    }
+  })
+
+  return nil
+}
+
+func (t *Translation) getEntry(tds *goquery.Selection) {
+  if tds == nil || tds.Length() < 4{
+    return
+  }
+
+  var (
+    entries []Entry
+    index int = 0
+  )
+
+  tds.Each(func(_ int,sel *goquery.Selection){
+    if index < 1 || index > 3 {
+      index++
+      return
+    }
+
+    link := sel.Find("a")
+    detail := sel.Find("i")
+    text := ""
+
+    if link.Length() == 0 {
+      text = sel.Text()
+    } else {
+      text = link.Text()
+    }
+
+    if detail == nil {
+      entries = append(entries, Entry{
+        Text: text,
+      })
+    }else {
+      entries = append(entries, Entry{
+        Text: text,
+        Detail: detail.Text(),
+      })
+    }
+
+    index++
+  })
+
+  t.Entries = append(t.Entries, entries)
+}
+
+func (t *Translation) getEntries(trs *goquery.Selection) {
+  trs.Each(func(_ int, sel *goquery.Selection){
+    tds := sel.Find("td")
+
+    if tds == nil {
+      return
+    }
+
+    if class, ok := tds.Last().Attr("class"); !ok {
+      return
+    }else if !strings.Contains(class, "rc4 hidden-xs"){
+      return
+    }
+
+    t.getEntry(tds)
+  })
+}
+
+func (t *Translation) GetResults(doc *goquery.Document) error {
+  table := doc.Find(".searchResultsTable")
+
+  if table.Length() == 0 {
+    return t.getSuggestions(doc)
+  }
+
+  trs := table.Find("tbody tr")
+
+  if trs.Length() == 0 {
+    return fmt.Errorf("failed to get table body")
+  }
+
+  trs.First().Find("th").Each(func(_ int, sel *goquery.Selection){
+    if sel.Text() != "" {
+      t.Fields = append(t.Fields, sel.Text())
+    }
+  })
+
+  t.getEntries(trs)
+  return nil
+}