1#include "test.h"
2
3#include <base/fs.h>
4#include <base/io.h>
5#include <base/str.h>
6
7#include <engine/shared/linereader.h>
8
9#include <gtest/gtest.h>
10
11void TestFileLineReaderRaw(const char *pWritten, unsigned WrittenLength, std::initializer_list<const char *> pExpectedLines, bool ExpectSuccess, bool WriteBom)
12{
13 CTestInfo Info;
14 IOHANDLE File = io_open(filename: Info.m_aFilename, flags: IOFLAG_WRITE);
15 ASSERT_TRUE(File);
16 if(WriteBom)
17 {
18 constexpr const unsigned char UTF8_BOM[] = {0xEF, 0xBB, 0xBF};
19 EXPECT_EQ(io_write(File, UTF8_BOM, sizeof(UTF8_BOM)), sizeof(UTF8_BOM));
20 }
21 EXPECT_EQ(io_write(File, pWritten, WrittenLength), WrittenLength);
22 EXPECT_FALSE(io_close(File));
23
24 CLineReader LineReader;
25 const bool ActualSuccess = LineReader.OpenFile(File: io_open(filename: Info.m_aFilename, flags: IOFLAG_READ));
26 ASSERT_EQ(ActualSuccess, ExpectSuccess);
27 if(ActualSuccess)
28 {
29 for(const char *pExpectedLine : pExpectedLines)
30 {
31 const char *pActualLine = LineReader.Get();
32 ASSERT_TRUE(pActualLine) << "Line reader returned less lines than expected. Expected next line: '" << pExpectedLine << "'";
33 EXPECT_STREQ(pActualLine, pExpectedLine) << "Line reader returned unexpected line";
34 }
35 const char *pActualLastLine = LineReader.Get();
36 EXPECT_FALSE(pActualLastLine) << "Line reader returned more lines than expected. Unexpected last line: '" << pActualLastLine << "'";
37 }
38
39 EXPECT_FALSE(fs_remove(Info.m_aFilename));
40}
41
42void TestFileLineReaderRaw(const char *pWritten, unsigned WrittenLength, std::initializer_list<const char *> pReads, bool ExpectSuccess)
43{
44 TestFileLineReaderRaw(pWritten, WrittenLength, pExpectedLines: pReads, ExpectSuccess, WriteBom: false);
45 TestFileLineReaderRaw(pWritten, WrittenLength, pExpectedLines: pReads, ExpectSuccess, WriteBom: true);
46}
47
48void TestFileLineReader(const char *pWritten, std::initializer_list<const char *> pReads)
49{
50 TestFileLineReaderRaw(pWritten, WrittenLength: str_length(str: pWritten), pReads, ExpectSuccess: true);
51}
52
53TEST(LineReader, LineFeedLineEndings)
54{
55 TestFileLineReader(pWritten: "foo\nbar\nbaz", pReads: {"foo", "bar", "baz"});
56 TestFileLineReader(pWritten: "foo\nbar\nbaz\n", pReads: {"foo", "bar", "baz"});
57}
58
59TEST(LineReader, CarriageReturnLineFeedLineEndings)
60{
61 TestFileLineReader(pWritten: "foo\r\nbar\r\nbaz", pReads: {"foo", "bar", "baz"});
62 TestFileLineReader(pWritten: "foo\r\nbar\r\nbaz\r\n", pReads: {"foo", "bar", "baz"});
63}
64
65TEST(LineReader, CarriageReturnLineEndings)
66{
67 // Line ending `\r` not supported
68 TestFileLineReader(pWritten: "foo\rbar\rbaz", pReads: {});
69 TestFileLineReader(pWritten: "foo\rbar\rbaz\r", pReads: {});
70}
71
72TEST(LineReader, MixedLineEndings)
73{
74 TestFileLineReader(pWritten: "1\n2\r\n3\n4\n5\r\n6", pReads: {"1", "2", "3", "4", "5", "6"});
75 TestFileLineReader(pWritten: "1\n2\r\n3\n4\n5\r\n6\n", pReads: {"1", "2", "3", "4", "5", "6"});
76 TestFileLineReader(pWritten: "1\n2\r\n3\n4\n5\r\n6\r\n", pReads: {"1", "2", "3", "4", "5", "6"});
77 TestFileLineReader(pWritten: "1\n2\r\n3\n4\n5\r\n6\r", pReads: {"1", "2", "3", "4", "5"}); // Line with trailing `\r` is skipped
78}
79
80TEST(LineReader, EmptyLines)
81{
82 TestFileLineReader(pWritten: "\n\r\n\n\n\r\n", pReads: {"", "", "", "", ""});
83 TestFileLineReader(pWritten: "\n\r\n\n\n\r\n\n", pReads: {"", "", "", "", "", ""});
84 TestFileLineReader(pWritten: "\n\r\n\n\n\r\n\r\n", pReads: {"", "", "", "", "", ""});
85 TestFileLineReader(pWritten: "\n\r\n\n\n\r\n\r", pReads: {"", "", "", "", ""}); // Line with trailing `\r` is skipped
86}
87
88TEST(LineReader, InvalidUtf8)
89{
90 // Lines containing invalid UTF-8 are skipped
91 TestFileLineReader(pWritten: "foo\xff\nbar\xff\nbaz\xff", pReads: {});
92 TestFileLineReader(pWritten: "foo\xff\nbar\nbaz", pReads: {"bar", "baz"});
93 TestFileLineReader(pWritten: "foo\nbar\xff\nbaz", pReads: {"foo", "baz"});
94 TestFileLineReader(pWritten: "foo\nbar\nbaz\xff", pReads: {"foo", "bar"});
95 TestFileLineReader(pWritten: "foo\nbar1\xff\nbar2\xff\nfoobar\nbar3\xff\nbaz", pReads: {"foo", "foobar", "baz"});
96}
97
98TEST(LineReader, ControlCharacters)
99{
100 // Lines containing control characters except `\t` are skipped
101 TestFileLineReader(
102 pWritten: "\x01\n\x02\n\x03\n\x04\n\x05\n\x06\n\x07\n\x08\n\x09\n\x0B\n\x0C\n\x0E\n\x0F\n\x10\n" // `\0x0A` and `\0x0D` are `\n` and `\r`
103 "\x11\n\x12\n\x13\n\x14\n\x15\n\x16\n\x17\n\x18\n\x19\n\x1A\n\x1B\n\x1C\n\x1D\n\x1E\n\x1F",
104 pReads: {"\t"});
105}
106
107TEST(LineReader, NullBytes)
108{
109 // Line reader does not read any lines if the file contains null bytes
110 TestFileLineReaderRaw(pWritten: "foo\0\nbar\nbaz", WrittenLength: 12, pReads: {}, ExpectSuccess: false);
111 TestFileLineReaderRaw(pWritten: "foo\nbar\0\nbaz", WrittenLength: 12, pReads: {}, ExpectSuccess: false);
112 TestFileLineReaderRaw(pWritten: "foo\nbar\nbaz\0", WrittenLength: 12, pReads: {}, ExpectSuccess: false);
113}
114