diff options
| -rw-r--r-- | main.go | 2 | ||||
| -rw-r--r-- | src/wikipedia/api.go | 17 | ||||
| -rw-r--r-- | src/wikipedia/links.gen.go | 10 | ||||
| -rw-r--r-- | src/wikipedia/wikipedia.go | 93 |
4 files changed, 122 insertions, 0 deletions
@@ -21,6 +21,7 @@ import ( "git.thomasvoss.com/euro-cash.eu/src/dbx" "git.thomasvoss.com/euro-cash.eu/src/email" "git.thomasvoss.com/euro-cash.eu/src/i18n" + "git.thomasvoss.com/euro-cash.eu/src/wikipedia" ) func main() { @@ -69,6 +70,7 @@ func main() { } i18n.Init(Try2(os.OpenRoot("po")).FS(), app.Debugp) + wikipedia.Init(i18n.DefaultPrinter.Bcp) dbx.Init(Try2(os.OpenRoot("src/dbx/sql")).FS()) app.BuildTemplates(Try2(os.OpenRoot("src/templates")).FS()) app.Run(*port) diff --git a/src/wikipedia/api.go b/src/wikipedia/api.go new file mode 100644 index 0000000..fb06518 --- /dev/null +++ b/src/wikipedia/api.go @@ -0,0 +1,17 @@ +package wikipedia + +type APIResponse struct { + Continue *struct { + Continue string `json:"continue"` + LlContinue string `json:"llcontinue"` + } `json:"continue"` + Query struct { + Pages []struct { + Title string `json:"title"` + LangLinks *[]struct { + Lang string `json:"lang"` + Title string `json:"title"` + } `json:"langlinks"` + } `json:"pages"` + } `json:"query"` +} diff --git a/src/wikipedia/links.gen.go b/src/wikipedia/links.gen.go new file mode 100644 index 0000000..2aa3c39 --- /dev/null +++ b/src/wikipedia/links.gen.go @@ -0,0 +1,10 @@ +package wikipedia + +/* TODO: Extract these programmatically */ + +var extractedTitles = [...]string{ + "Coat_of_arms_of_Andorra", + "Coat_of_arms_of_Bulgaria", + "Coat_of_arms_of_Croatia", + "Coat_of_arms_of_Germany", +} diff --git a/src/wikipedia/wikipedia.go b/src/wikipedia/wikipedia.go new file mode 100644 index 0000000..a77f765 --- /dev/null +++ b/src/wikipedia/wikipedia.go @@ -0,0 +1,93 @@ +package wikipedia + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" +) + +var ( + defaultLocale string + titlemap = make(map[string]map[string]string) +) + +func Init(locale string) { + defaultLocale = locale + base := fmt.Sprintf("https://%s.wikipedia.org/w/api.php", defaultLocale) + u, err := url.Parse(base) + if err != nil { + log.Println(err) + return + } + + var resp APIResponse + titles := strings.Join(extractedTitles[:], "|") + + q := u.Query() + q.Set("action", "query") + q.Set("format", "json") + q.Set("prop", "langlinks") + q.Set("titles", titles) + q.Set("formatversion", "2") + q.Set("lllimit", "100") + + for { + if resp.Continue != nil { + q.Set("continue", resp.Continue.Continue) + q.Set("llcontinue", resp.Continue.LlContinue) + } + u.RawQuery = q.Encode() + + respjson, err := http.Get(u.String()) + if err != nil { + log.Println(err) + return + } + defer respjson.Body.Close() + + body, err := io.ReadAll(respjson.Body) + if err != nil { + log.Println(err) + return + } + + resp = APIResponse{} + if err = json.Unmarshal(body, &resp); err != nil { + log.Println(err) + return + } + + for _, page := range resp.Query.Pages { + if page.LangLinks == nil { + continue + } + + t := url.PathEscape(page.Title) + if _, ok := titlemap[t]; !ok { + titlemap[t] = make(map[string]string) + } + + for _, ll := range *page.LangLinks { + titlemap[t][ll.Lang] = url.PathEscape(ll.Title) + } + } + + if resp.Continue == nil { + break + } + } +} + +func Url(title, locale string) string { + title = url.PathEscape(title) + t, ok := titlemap[title][locale] + if !ok { + return fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", + defaultLocale, title) + } + return fmt.Sprintf("https://%s.wikipedia.org/wiki/%s", locale, t) +} |