1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
package wikipedia
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
var (
defaultLocale string
titlemap = make(map[string]map[string]string)
)
func Init(locale string) error {
defaultLocale = locale
base := fmt.Sprintf("https://%s.wikipedia.org/w/api.php", defaultLocale)
u, err := url.Parse(base)
if err != nil {
return err
}
var resp APIResponse
titles := strings.Join(extractedTitles[:], "|")
q := u.Query()
q.Set("action", "query")
q.Set("format", "json")
q.Set("prop", "langlinks")
q.Set("titles", titles)
q.Set("formatversion", "2")
q.Set("lllimit", "max")
for {
if resp.Continue != nil {
q.Set("continue", resp.Continue.Continue)
q.Set("llcontinue", resp.Continue.LlContinue)
}
u.RawQuery = q.Encode()
/* TODO: Use a context and NewRequestWithContext()? */
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", "euro-cash.eu/1.0.0 (admin@euro-cash.eu)")
respjson, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
if respjson.StatusCode >= 400 &&
respjson.StatusCode != http.StatusTooManyRequests {
msg := respjson.Status
bytes, err := io.ReadAll(respjson.Body)
if err == nil {
msg = string(bytes)
}
return fmt.Errorf("Failed to GET %s: %s", u, msg)
}
defer respjson.Body.Close()
secs, err := strconv.Atoi(respjson.Header.Get("Retry-After"))
if err != nil {
time.Sleep(time.Duration(secs) * time.Second)
}
body, err := io.ReadAll(respjson.Body)
if err != nil {
return err
}
resp = APIResponse{}
if err = json.Unmarshal(body, &resp); err != nil {
return err
}
for _, page := range resp.Query.Pages {
if page.LangLinks == nil {
continue
}
t := url.PathEscape(page.Title)
if _, ok := titlemap[t]; !ok {
titlemap[t] = make(map[string]string)
}
for _, ll := range *page.LangLinks {
titlemap[t][ll.Lang] = url.PathEscape(ll.Title)
}
}
if resp.Continue == nil {
return nil
}
}
}
func Url(title, locale string) string {
base := "https://%s.wikipedia.org/wiki/%s"
title = url.PathEscape(title)
t, ok := titlemap[title][locale]
if !ok {
t, locale = title, defaultLocale
}
return fmt.Sprintf(base, locale, t)
}
|
