1/* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
2/* If you are missing that file, acquire a complete release at teeworlds.com. */
3
4#include "localization.h"
5
6#include <engine/console.h>
7#include <engine/shared/linereader.h>
8#include <engine/storage.h>
9
10const char *Localize(const char *pStr, const char *pContext)
11{
12 const char *pNewStr = g_Localization.FindString(Hash: str_quickhash(str: pStr), ContextHash: str_quickhash(str: pContext));
13 return pNewStr ? pNewStr : pStr;
14}
15
16void CLocalizationDatabase::LoadIndexfile(IStorage *pStorage, IConsole *pConsole)
17{
18 m_vLanguages.clear();
19
20 const std::vector<std::string> vEnglishLanguageCodes = {"en"};
21 m_vLanguages.emplace_back(args: "English", args: "", args: 826, args: vEnglishLanguageCodes);
22
23 const char *pFilename = "languages/index.txt";
24 IOHANDLE File = pStorage->OpenFile(pFilename, Flags: IOFLAG_READ | IOFLAG_SKIP_BOM, Type: IStorage::TYPE_ALL);
25 if(!File)
26 {
27 char aBuf[64 + IO_MAX_PATH_LENGTH];
28 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "Couldn't open index file '%s'", pFilename);
29 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
30 return;
31 }
32
33 CLineReader LineReader;
34 LineReader.Init(File);
35
36 const char *pLine;
37 while((pLine = LineReader.Get()))
38 {
39 if(!str_length(str: pLine) || pLine[0] == '#') // skip empty lines and comments
40 continue;
41
42 char aEnglishName[128];
43 str_copy(dst&: aEnglishName, src: pLine);
44
45 pLine = LineReader.Get();
46 if(!pLine)
47 {
48 char aBuf[256];
49 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "Unexpected end of index file after language '%s'", aEnglishName);
50 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
51 break;
52 }
53 if(!str_startswith(str: pLine, prefix: "== "))
54 {
55 char aBuf[256];
56 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "Missing native name for language '%s'", aEnglishName);
57 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
58 (void)LineReader.Get();
59 (void)LineReader.Get();
60 continue;
61 }
62 char aNativeName[128];
63 str_copy(dst&: aNativeName, src: pLine + 3);
64
65 pLine = LineReader.Get();
66 if(!pLine)
67 {
68 char aBuf[256];
69 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "Unexpected end of index file after language '%s'", aEnglishName);
70 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
71 break;
72 }
73 if(!str_startswith(str: pLine, prefix: "== "))
74 {
75 char aBuf[256];
76 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "Missing country code for language '%s'", aEnglishName);
77 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
78 (void)LineReader.Get();
79 continue;
80 }
81 char aCountryCode[128];
82 str_copy(dst&: aCountryCode, src: pLine + 3);
83
84 pLine = LineReader.Get();
85 if(!pLine)
86 {
87 char aBuf[256];
88 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "Unexpected end of index file after language '%s'", aEnglishName);
89 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
90 break;
91 }
92 if(!str_startswith(str: pLine, prefix: "== "))
93 {
94 char aBuf[256];
95 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "Missing language codes for language '%s'", aEnglishName);
96 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
97 continue;
98 }
99 const char *pLanguageCodes = pLine + 3;
100 char aLanguageCode[256];
101 std::vector<std::string> vLanguageCodes;
102 while((pLanguageCodes = str_next_token(str: pLanguageCodes, delim: ";", buffer: aLanguageCode, buffer_size: sizeof(aLanguageCode))))
103 {
104 if(aLanguageCode[0])
105 {
106 vLanguageCodes.emplace_back(args&: aLanguageCode);
107 }
108 }
109 if(vLanguageCodes.empty())
110 {
111 char aBuf[256];
112 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "At least one language code required for language '%s'", aEnglishName);
113 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
114 continue;
115 }
116
117 char aFileName[IO_MAX_PATH_LENGTH];
118 str_format(buffer: aFileName, buffer_size: sizeof(aFileName), format: "languages/%s.txt", aEnglishName);
119 m_vLanguages.emplace_back(args&: aNativeName, args&: aFileName, args: str_toint(str: aCountryCode), args&: vLanguageCodes);
120 }
121
122 io_close(io: File);
123
124 std::sort(first: m_vLanguages.begin(), last: m_vLanguages.end());
125}
126
127void CLocalizationDatabase::SelectDefaultLanguage(IConsole *pConsole, char *pFilename, size_t Length) const
128{
129 if(Languages().empty())
130 return;
131 if(Languages().size() == 1)
132 {
133 str_copy(dst: pFilename, src: Languages()[0].m_FileName.c_str(), dst_size: Length);
134 return;
135 }
136
137 char aLocaleStr[128];
138 os_locale_str(locale: aLocaleStr, length: sizeof(aLocaleStr));
139
140 char aBuf[256];
141 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "Choosing default language based on user locale '%s'", aLocaleStr);
142 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
143
144 while(true)
145 {
146 const CLanguage *pPrefixMatch = nullptr;
147 for(const auto &Language : Languages())
148 {
149 for(const auto &LanguageCode : Language.m_vLanguageCodes)
150 {
151 if(LanguageCode == aLocaleStr)
152 {
153 // Exact match found, use it immediately
154 str_copy(dst: pFilename, src: Language.m_FileName.c_str(), dst_size: Length);
155 return;
156 }
157 else if(LanguageCode.rfind(s: aLocaleStr, pos: 0) == 0)
158 {
159 // Locale is prefix of language code, e.g. locale is "en" and current language is "en-US"
160 pPrefixMatch = &Language;
161 }
162 }
163 }
164 // Use prefix match if no exact match was found
165 if(pPrefixMatch)
166 {
167 str_copy(dst: pFilename, src: pPrefixMatch->m_FileName.c_str(), dst_size: Length);
168 return;
169 }
170
171 // Remove last segment of locale string and try again with more generic locale, e.g. "en-US" -> "en"
172 int i = str_length(str: aLocaleStr) - 1;
173 for(; i >= 0; --i)
174 {
175 if(aLocaleStr[i] == '-')
176 {
177 aLocaleStr[i] = '\0';
178 break;
179 }
180 }
181
182 // Stop if no more locale segments are left
183 if(i <= 0)
184 break;
185 }
186}
187
188bool CLocalizationDatabase::Load(const char *pFilename, IStorage *pStorage, IConsole *pConsole)
189{
190 // empty string means unload
191 if(pFilename[0] == 0)
192 {
193 m_vStrings.clear();
194 m_StringsHeap.Reset();
195 return true;
196 }
197
198 IOHANDLE IoHandle = pStorage->OpenFile(pFilename, Flags: IOFLAG_READ | IOFLAG_SKIP_BOM, Type: IStorage::TYPE_ALL);
199 if(!IoHandle)
200 return false;
201
202 char aBuf[256];
203 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "loaded '%s'", pFilename);
204 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
205 m_vStrings.clear();
206 m_StringsHeap.Reset();
207
208 char aContext[512];
209 char aOrigin[512];
210 CLineReader LineReader;
211 LineReader.Init(File: IoHandle);
212 char *pLine;
213 int Line = 0;
214 while((pLine = LineReader.Get()))
215 {
216 Line++;
217 if(!str_length(str: pLine))
218 continue;
219
220 if(pLine[0] == '#') // skip comments
221 continue;
222
223 if(pLine[0] == '[') // context
224 {
225 size_t Len = str_length(str: pLine);
226 if(Len < 1 || pLine[Len - 1] != ']')
227 {
228 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "malformed context line (%d): %s", Line, pLine);
229 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
230 continue;
231 }
232 str_copy(dst: aContext, src: pLine + 1, dst_size: Len - 1);
233 pLine = LineReader.Get();
234 }
235 else
236 {
237 aContext[0] = '\0';
238 }
239
240 str_copy(dst: aOrigin, src: pLine, dst_size: sizeof(aOrigin));
241 char *pReplacement = LineReader.Get();
242 Line++;
243 if(!pReplacement)
244 {
245 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: "unexpected end of file");
246 break;
247 }
248
249 if(pReplacement[0] != '=' || pReplacement[1] != '=' || pReplacement[2] != ' ')
250 {
251 str_format(buffer: aBuf, buffer_size: sizeof(aBuf), format: "malformed replacement line (%d) for '%s'", Line, aOrigin);
252 pConsole->Print(Level: IConsole::OUTPUT_LEVEL_ADDINFO, pFrom: "localization", pStr: aBuf);
253 continue;
254 }
255
256 pReplacement += 3;
257 AddString(pOrgStr: aOrigin, pNewStr: pReplacement, pContext: aContext);
258 }
259 io_close(io: IoHandle);
260 std::sort(first: m_vStrings.begin(), last: m_vStrings.end());
261 return true;
262}
263
264void CLocalizationDatabase::AddString(const char *pOrgStr, const char *pNewStr, const char *pContext)
265{
266 m_vStrings.emplace_back(args: str_quickhash(str: pOrgStr), args: str_quickhash(str: pContext), args: m_StringsHeap.StoreString(pSrc: *pNewStr ? pNewStr : pOrgStr));
267}
268
269const char *CLocalizationDatabase::FindString(unsigned Hash, unsigned ContextHash) const
270{
271 CString String;
272 String.m_Hash = Hash;
273 String.m_ContextHash = ContextHash;
274 String.m_pReplacement = 0x0;
275 auto Range1 = std::equal_range(first: m_vStrings.begin(), last: m_vStrings.end(), val: String);
276 if(std::distance(first: Range1.first, last: Range1.second) == 1)
277 return Range1.first->m_pReplacement;
278
279 const unsigned DefaultHash = str_quickhash(str: "");
280 if(ContextHash != DefaultHash)
281 {
282 // Do another lookup with the default context hash
283 String.m_ContextHash = DefaultHash;
284 auto Range2 = std::equal_range(first: m_vStrings.begin(), last: m_vStrings.end(), val: String);
285 if(std::distance(first: Range2.first, last: Range2.second) == 1)
286 return Range2.first->m_pReplacement;
287 }
288
289 return nullptr;
290}
291
292CLocalizationDatabase g_Localization;
293