diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -150,6 +150,9 @@ let Macros = [ SimpleMacroDef<"stderr", "stderr">, SimpleMacroDef<"stdout", "stdout">, + SimpleMacroDef<"_IOFBF", "0">, + SimpleMacroDef<"_IOLBF", "1">, + SimpleMacroDef<"_IONBF", "2">, ]; let Types = ["size_t", "FILE", "cookie_io_functions_t"]; } diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -474,6 +474,9 @@ [ Macro<"stderr">, Macro<"stdout">, + Macro<"_IOFBF">, + Macro<"_IOLBF">, + Macro<"_IONBF">, ], // Macros [ // Types SizeTType, diff --git a/libc/src/__support/File/file.cpp b/libc/src/__support/File/file.cpp --- a/libc/src/__support/File/file.cpp +++ b/libc/src/__support/File/file.cpp @@ -25,52 +25,151 @@ prev_op = FileOp::WRITE; + if (bufmode == _IONBF) { // unbuffered + size_t written = platform_write(this, data, len); + if (written < len) + err = true; + platform_flush(this); + return written; + } + cpp::ArrayRef dataref(data, len); cpp::MutableArrayRef bufref(buf, bufsize); const size_t used = pos; const size_t bufspace = bufsize - pos; - const size_t write_size = bufspace > len ? len : bufspace; - // TODO: Replace the for loop below with a call to internal memcpy. - for (size_t i = 0; i < write_size; ++i) - bufref[pos + i] = dataref[i]; - pos += write_size; - if (len < bufspace) - return len; - // If the control reaches beyond this point, it means that |data| - // is more than what can be accomodated in the buffer. So, we first - // flush out the buffer. - size_t bytes_written = platform_write(this, buf, bufsize); - pos = 0; // Buffer is now empty so reset pos to the beginning. - if (bytes_written < bufsize) { - err = true; - // If less bytes were written than expected, then there are two - // possibilities. - // 1. None of the bytes from |data| were flushed out. - if (bytes_written <= used) - return 0; - // 2. Some of the bytes from |data| were written - return bytes_written - used; + // The flush point is the number of characters that can be written from data + // before the buffer must be flushed. In fully buffered mode, this is always + // the remaining size of the buffer, but in line buffered mode it can be the + // index of the last newline in data. The characters after the flush point may + // be buffered if there's space. + size_t flush_point = bufspace; + + /* + Some examples to demonstrate the flush point: + + buffer[8] = [________] + data[4] = [abcd] + + in this case the flush point is at 8, since the two flush conditions are + hitting a newline (in line buffered mode) and hitting the end of the buffer. + Since the flush point is beyond the end of where data is writing to, no flush + occurs. + Result is no output with buffer [abcd____] + + + buffer[8] = [12345___] + data[4] = [abcd] + + in this case the flush point is at 3, since there are three more spaces left + in buffer before it needs a flush. + Result is [12345abc] with buffer [d_______] + + + buffer[8] = [12345___] + data[5] = [A\nBCD] + + since there's a newline, flush point is 2, since we always want to + flush up to the newline. The newline fits into the buffer, so we copy up to + that point into the buffer, then flush the buffer before copying the remainder + in. + Result is [12345A\n] with buffer [BCD_____] + + + buffer[8] = [12345___] + data[7] = [ABCD\nEF] + + There's a newline again, but now it won't fit into the buffer. The flush point + is 5, but there's only 3 spaces left in the buffer. The solution is to flush + the buffer first, then write up to the flush point directly from data. Since + there's enough space in the buffer for the remainder, it's copied into the + buffer. + Result is two writes, [12345] and [ABCD\n] with buffer [EF______] + + + buffer[8] = [12345___] + data[14] = [ABCD\n987654321] + + For this final example, the flush point (which is again 5) doesn't fit in the + buffer, and neither does the remainder. This means that the solution is to + flush the buffer and then write the whole of data directly, unbuffered. + Result is two writes, [12345] and [ABCD\n987654321] with an empty buffer + */ + + if (bufmode == _IOLBF) { + constexpr char NEWLINE_CHAR = '\n'; + size_t last_newline = len; + for (size_t i = len - 1; i > 0; --i) { + if (reinterpret_cast(data)[i] == NEWLINE_CHAR) { + last_newline = i; + break; + } + } + if (last_newline != len) { + flush_point = last_newline + 1; + } } - // If the remaining bytes from |data| can fit in the buffer, write - // into it. Else, write it directly to the platform stream. - size_t remaining = len - write_size; - if (remaining <= len) { + // this is the number of characters to be copied from data to buf. + size_t write_size = flush_point > len ? len : flush_point; + + // if there's enough space in the buffer to copy those characters, then do. + if (write_size <= bufspace) { // TODO: Replace the for loop below with a call to internal memcpy. - for (size_t i = 0; i < remaining; ++i) - bufref[i] = dataref[i]; - pos += remaining; - return len; + for (size_t i = 0; i < write_size; ++i) + bufref[pos + i] = dataref[i]; + pos += write_size; + // if the flush point has not been hit, then no flush is necessary. + if (len < flush_point) + return len; + } else + write_size = 0; // else no characters were copied. + + // If the control reaches beyond this point, it means that the flush point has + // been hit, so first we flush the buffer. + if (pos > 0) { + size_t bytes_in_buff = pos; + size_t bytes_written = platform_write(this, buf, bytes_in_buff); + pos = 0; // Buffer is now empty so reset pos to the beginning. + if (bytes_written < bytes_in_buff) { + err = true; + // If less bytes were written than expected, then there are two + // possibilities. + // 1. None of the bytes from |data| were flushed out. + if (bytes_written <= used) + return 0; + // 2. Some of the bytes from |data| were written + return bytes_written - used; + } } - size_t transferred = - platform_write(this, dataref.data() + write_size, remaining); - if (transferred < remaining) { - err = true; - return write_size + transferred; + size_t remaining = len - flush_point; + + // if the flush point is after the end of the buffer, then we will need to + // write some characters from data directly. + if (flush_point > bufspace) { + // if the remaining characters after the flush point won't fit in the + // buffer, write all of the characters. + size_t rem_write_size = + (remaining > bufsize ? len : flush_point) - write_size; + size_t transferred = + platform_write(this, dataref.data() + write_size, rem_write_size); + if (transferred < remaining) { + err = true; + return write_size + transferred; + } } + + // if the remaining characters after the flush point will fit in the buffer, + // copy them into the buffer. + if (remaining <= bufsize) { + // TODO: Replace the for loop below with a call to internal memcpy. + for (size_t i = 0; i < remaining; ++i) + bufref[i] = dataref[i + flush_point]; + pos = remaining; // to reach this point, the buffer must have been flushed. + } + return len; } diff --git a/libc/src/__support/File/linux_file.cpp b/libc/src/__support/File/linux_file.cpp --- a/libc/src/__support/File/linux_file.cpp +++ b/libc/src/__support/File/linux_file.cpp @@ -11,7 +11,8 @@ #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include -#include // For mode_t and other flags to the open syscall +#include // For mode_t and other flags to the open syscall +#include #include // For malloc #include // For syscall numbers @@ -158,24 +159,19 @@ void *buffer = malloc(File::DEFAULT_BUFFER_SIZE); auto *file = reinterpret_cast(malloc(sizeof(LinuxFile))); - LinuxFile::init( - file, fd, buffer, File::DEFAULT_BUFFER_SIZE, - 0, // TODO: Set the correct buffer mode when buffer mode is available. - true, modeflags); + LinuxFile::init(file, fd, buffer, File::DEFAULT_BUFFER_SIZE, _IOFBF, true, + modeflags); return file; } -// TODO: Use the appropriate buffering modes for the standard streams below -// the different buffering modes are available. constexpr size_t STDOUT_BUFFER_SIZE = 1024; char stdout_buffer[STDOUT_BUFFER_SIZE]; -static LinuxFile StdOut(1, stdout_buffer, STDOUT_BUFFER_SIZE, 0, false, +static LinuxFile StdOut(1, stdout_buffer, STDOUT_BUFFER_SIZE, _IOLBF, false, File::ModeFlags(File::OpenMode::APPEND)); File *stdout = &StdOut; -constexpr size_t STDERR_BUFFER_SIZE = 1024; -char stderr_buffer[STDERR_BUFFER_SIZE]; -static LinuxFile StdErr(2, stderr_buffer, STDERR_BUFFER_SIZE, 0, false, +constexpr size_t STDERR_BUFFER_SIZE = 0; +static LinuxFile StdErr(2, nullptr, STDERR_BUFFER_SIZE, _IONBF, false, File::ModeFlags(File::OpenMode::APPEND)); File *stderr = &StdErr; diff --git a/libc/test/src/__support/File/file_test.cpp b/libc/test/src/__support/File/file_test.cpp --- a/libc/test/src/__support/File/file_test.cpp +++ b/libc/test/src/__support/File/file_test.cpp @@ -116,7 +116,8 @@ const char data[] = "hello, file"; constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2; char file_buffer[FILE_BUFFER_SIZE]; - StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "w"); + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "w"); ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); EXPECT_EQ(f->get_pos(), size_t(0)); // Data is buffered in the file stream @@ -133,6 +134,9 @@ EXPECT_GE(f->get_pos(), size_t(0)); ASSERT_EQ(f->flush(), 0); EXPECT_EQ(f->get_pos(), 2 * sizeof(data)); + MemoryView src1("hello, file\0hello, file", sizeof(data) * 2), + dst1(f->get_str(), sizeof(data) * 2); + EXPECT_MEM_EQ(src1, dst1); char read_data[sizeof(data)]; // This is not a readable file. @@ -144,11 +148,58 @@ ASSERT_EQ(f->close(), 0); } +TEST(LlvmLibcFileTest, WriteLineBuffered) { + const char data[] = "hello\n file"; + constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2; + char file_buffer[FILE_BUFFER_SIZE]; + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOLBF, false, "w"); + + ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); + EXPECT_EQ(f->get_pos(), size_t(6)); // buffer after the newline + MemoryView src1("hello\n", 6), dst1(f->get_str(), 6); + EXPECT_MEM_EQ(src1, dst1); + + const char data2[] = "longer for an \n overflow"; + + ASSERT_EQ(sizeof(data2), f->write(data2, sizeof(data2))); + // the buffer's initial contents should be " file\0" + // Writing data2 should cause two flushes, the first being when the buffer + // fills, flushing " file\0longer for a", then the second being when a newline + // is hit, flushing "n \n" and leaving " overflow\0" in the buffer. + EXPECT_EQ(f->get_pos(), size_t(27)); + MemoryView src2("hello\n file\0longer for an \n", 27), dst2(f->get_str(), 27); + EXPECT_MEM_EQ(src2, dst2); + ASSERT_EQ(f->flush(), 0); + EXPECT_EQ(f->get_pos(), sizeof(data) + sizeof(data2)); + MemoryView src3("hello\n file\0longer for an \n overflow", 37), + dst3(f->get_str(), 37); + EXPECT_MEM_EQ(src3, dst3); + + ASSERT_EQ(f->close(), 0); +} + +TEST(LlvmLibcFileTest, WriteUnbuffered) { + const char data[] = "written immediately"; + constexpr size_t FILE_BUFFER_SIZE = sizeof(data) + 1; + char file_buffer[FILE_BUFFER_SIZE]; + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IONBF, false, "w"); + + ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); + EXPECT_EQ(f->get_pos(), + sizeof(data)); // no buffering means this is written immediately. + EXPECT_STREQ(f->get_str(), data); + + ASSERT_EQ(f->close(), 0); +} + TEST(LlvmLibcFileTest, ReadOnly) { const char initial_content[] = "1234567890987654321"; constexpr size_t FILE_BUFFER_SIZE = sizeof(initial_content); char file_buffer[FILE_BUFFER_SIZE]; - StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "r"); + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "r"); f->reset_and_fill(initial_content, sizeof(initial_content)); constexpr size_t READ_SIZE = sizeof(initial_content) / 2; @@ -198,7 +249,8 @@ const char initial_content[] = "1234567890987654321"; constexpr size_t FILE_BUFFER_SIZE = sizeof(initial_content); char file_buffer[FILE_BUFFER_SIZE]; - StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "r"); + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "r"); f->reset_and_fill(initial_content, sizeof(initial_content)); constexpr size_t READ_SIZE = 5; @@ -220,7 +272,8 @@ const char write_data[] = "append"; constexpr size_t FILE_BUFFER_SIZE = sizeof(write_data) * 3 / 2; char file_buffer[FILE_BUFFER_SIZE]; - StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "a"); + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "a"); f->reset_and_fill(initial_content, sizeof(initial_content)); constexpr size_t READ_SIZE = 5; @@ -246,7 +299,7 @@ constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2; char file_buffer[FILE_BUFFER_SIZE]; StringFile *f = - new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "w+"); + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "w+"); ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); EXPECT_EQ(f->get_pos(), size_t(0)); // Data is buffered in the file stream @@ -266,7 +319,7 @@ constexpr size_t FILE_BUFFER_SIZE = sizeof(initial_content); char file_buffer[FILE_BUFFER_SIZE]; StringFile *f = - new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "r+"); + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "r+"); f->reset_and_fill(initial_content, sizeof(initial_content)); constexpr size_t READ_SIZE = sizeof(initial_content) / 2; @@ -300,7 +353,7 @@ constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2; char file_buffer[FILE_BUFFER_SIZE]; StringFile *f = - new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "a+"); + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "a+"); f->reset_and_fill(initial_content, sizeof(initial_content)); ASSERT_EQ(sizeof(data), f->write(data, sizeof(data)));