diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td --- a/libc/config/linux/api.td +++ b/libc/config/linux/api.td @@ -150,6 +150,9 @@ let Macros = [ SimpleMacroDef<"stderr", "stderr">, SimpleMacroDef<"stdout", "stdout">, + SimpleMacroDef<"_IOFBF", "0">, + SimpleMacroDef<"_IOLBF", "1">, + SimpleMacroDef<"_IONBF", "2">, ]; let Types = ["size_t", "FILE", "cookie_io_functions_t"]; } diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td --- a/libc/spec/stdc.td +++ b/libc/spec/stdc.td @@ -474,6 +474,9 @@ [ Macro<"stderr">, Macro<"stdout">, + Macro<"_IOFBF">, + Macro<"_IOLBF">, + Macro<"_IONBF">, ], // Macros [ // Types SizeTType, diff --git a/libc/src/__support/File/file.h b/libc/src/__support/File/file.h --- a/libc/src/__support/File/file.h +++ b/libc/src/__support/File/file.h @@ -217,6 +217,10 @@ // Returns an bit map of flags corresponding to enumerations of // OpenMode, ContentType and CreateType. static ModeFlags mode_flags(const char *mode); + +private: + size_t write_unlocked_lbf(const void *data, size_t len); + size_t write_unlocked_fbf(const void *data, size_t len); }; // The implementaiton of this function is provided by the platfrom_file diff --git a/libc/src/__support/File/file.cpp b/libc/src/__support/File/file.cpp --- a/libc/src/__support/File/file.cpp +++ b/libc/src/__support/File/file.cpp @@ -25,52 +25,177 @@ prev_op = FileOp::WRITE; - cpp::ArrayRef dataref(data, len); - cpp::MutableArrayRef bufref(buf, bufsize); + if (bufmode == _IOFBF) { // fully buffered + return write_unlocked_fbf(data, len); + } else if (bufmode == _IOLBF) { // line buffered + return write_unlocked_lbf(data, len); + } else /*if (bufmode == _IONBF) */ { // unbuffered + size_t written = platform_write(this, data, len); + if (written < len) + err = true; + platform_flush(this); + return written; + } +} - const size_t used = pos; +size_t File::write_unlocked_fbf(const void *data, size_t len) { + const size_t init_pos = pos; const size_t bufspace = bufsize - pos; - const size_t write_size = bufspace > len ? len : bufspace; + + // we split |data| (conceptually) using the split point. Then we handle the + // two pieces separately. + const size_t split_point = len < bufspace ? len : bufspace; + + // The primary piece is the piece of |data| we want to write to the buffer + // before flushing. It will always fit into the buffer, since the split point + // is defined as being min(len, bufspace), and it will always exist if len is + // non-zero. + cpp::ArrayRef primary(data, split_point); + + // The second piece is the remainder of |data|. It is written to the buffer if + // it fits, or written directly to the output if it doesn't. If the primary + // piece fits entirely in the buffer, the remainder may be nothing. + cpp::ArrayRef remainder( + static_cast(data) + split_point, len - split_point); + + cpp::MutableArrayRef bufref(buf, bufsize); + + // Copy the first piece into the buffer. // TODO: Replace the for loop below with a call to internal memcpy. - for (size_t i = 0; i < write_size; ++i) - bufref[pos + i] = dataref[i]; - pos += write_size; - if (len < bufspace) + for (size_t i = 0; i < primary.size(); ++i) + bufref[pos + i] = primary[i]; + pos += primary.size(); + + // If there is no remainder, we can return early, since the first piece has + // fit completely into the buffer. + if (remainder.size() == 0) return len; - // If the control reaches beyond this point, it means that |data| - // is more than what can be accomodated in the buffer. So, we first - // flush out the buffer. - size_t bytes_written = platform_write(this, buf, bufsize); + // We need to flush the buffer now, since there is still data and the buffer + // is full. + const size_t write_size = pos; + size_t bytes_written = platform_write(this, buf, write_size); pos = 0; // Buffer is now empty so reset pos to the beginning. - if (bytes_written < bufsize) { + // If less bytes were written than expected, then an error occurred. Return + // the number of bytes that have been written from |data|. + if (bytes_written < write_size) { err = true; - // If less bytes were written than expected, then there are two - // possibilities. - // 1. None of the bytes from |data| were flushed out. - if (bytes_written <= used) - return 0; - // 2. Some of the bytes from |data| were written - return bytes_written - used; + return bytes_written <= init_pos ? 0 : bytes_written - init_pos; } - // If the remaining bytes from |data| can fit in the buffer, write - // into it. Else, write it directly to the platform stream. - size_t remaining = len - write_size; - if (remaining <= len) { + // The second piece is handled basically the same as the first, although we + // know that if the second piece has data in it then the buffer has been + // flushed, meaning that pos is always 0. + if (remainder.size() < bufsize) { // TODO: Replace the for loop below with a call to internal memcpy. - for (size_t i = 0; i < remaining; ++i) - bufref[i] = dataref[i]; - pos += remaining; - return len; + for (size_t i = 0; i < remainder.size(); ++i) + bufref[i] = remainder[i]; + pos = remainder.size(); + } else { + size_t bytes_written = + platform_write(this, remainder.data(), remainder.size()); + + // If less bytes were written than expected, then an error occurred. Return + // the number of bytes that have been written from |data|. + if (bytes_written < remainder.size()) { + err = true; + return primary.size() + bytes_written; + } } - size_t transferred = - platform_write(this, dataref.data() + write_size, remaining); - if (transferred < remaining) { + return len; +} + +size_t File::write_unlocked_lbf(const void *data, size_t len) { + const size_t init_pos = pos; + const size_t bufspace = bufsize - pos; + + constexpr char NEWLINE_CHAR = '\n'; + size_t last_newline = len; + for (size_t i = len - 1; i > 0; --i) { + if (static_cast(data)[i] == NEWLINE_CHAR) { + last_newline = i; + break; + } + } + if (last_newline == len) { + return write_unlocked_fbf(data, len); + } + + // we split |data| (conceptually) using the split point. Then we handle the + // two pieces separately. + const size_t split_point = last_newline + 1; + + // The primary piece is everything in |data| up to the newline. If it fits in + // the buffer, then we'll write it there before flushing the buffer. + // Otherwise, we need to write it to the output directly. + cpp::ArrayRef primary(data, split_point); + + // The second piece is the remainder of |data|. It is written to the buffer if + // it fits, or written directly to the output if it doesn't. + cpp::ArrayRef remainder( + static_cast(data) + split_point, len - split_point); + + cpp::MutableArrayRef bufref(buf, bufsize); + + // If the primary piece fits in the buffer, copy the it into the buffer. + if (primary.size() <= bufspace) { + // TODO: Replace the for loop below with a call to internal memcpy. + for (size_t i = 0; i < primary.size(); ++i) + bufref[pos + i] = primary[i]; + pos += primary.size(); + } + + // Since this is line buffered, and we only get here if there's a newline, we + // always flush the buffer. + const size_t write_size = pos; + size_t bytes_written = platform_write(this, buf, write_size); + pos = 0; // Buffer is now empty so reset pos to the beginning. + + // If less bytes were written than expected, then an error occurred. Return + // the number of bytes that have been written from |data|. + if (bytes_written < write_size) { err = true; - return write_size + transferred; + return bytes_written <= init_pos ? 0 : bytes_written - init_pos; + } + + // If the primary piece doesn't fit in the buffer, write it directly to the + // output. This isn't an else for the first condition because it must happen + // after the buffer is flushed. + if (primary.size() > bufspace) { + size_t bytes_written = platform_write(this, primary.data(), primary.size()); + + // If less bytes were written than expected, then an error occurred. Return + // the number of bytes that have been written from |data|. + if (bytes_written < primary.size()) { + err = true; + return bytes_written; + } + } + + // The remainder is handled basically the same as the primary, although we + // know that if the remainder has data in it then the buffer has been + // flushed, meaning that pos is always 0. + if (remainder.size() < bufsize) { + // TODO: Replace the for loop below with a call to internal memcpy. + for (size_t i = 0; i < remainder.size(); ++i) + bufref[i] = remainder[i]; + pos = remainder.size(); + } else { + size_t bytes_written = + platform_write(this, remainder.data(), remainder.size()); + + // If less bytes were written than expected, then an error occurred. Return + // the number of bytes that have been written from |data|. + if (bytes_written < remainder.size()) { + err = true; + return primary.size() + bytes_written; + } } + + // This is in line buffered mode, so the data should be fully flushed. + platform_flush(this); + return len; } diff --git a/libc/src/__support/File/linux_file.cpp b/libc/src/__support/File/linux_file.cpp --- a/libc/src/__support/File/linux_file.cpp +++ b/libc/src/__support/File/linux_file.cpp @@ -11,7 +11,8 @@ #include "src/__support/OSUtil/syscall.h" // For internal syscall function. #include -#include // For mode_t and other flags to the open syscall +#include // For mode_t and other flags to the open syscall +#include #include // For malloc #include // For syscall numbers @@ -158,24 +159,19 @@ void *buffer = malloc(File::DEFAULT_BUFFER_SIZE); auto *file = reinterpret_cast(malloc(sizeof(LinuxFile))); - LinuxFile::init( - file, fd, buffer, File::DEFAULT_BUFFER_SIZE, - 0, // TODO: Set the correct buffer mode when buffer mode is available. - true, modeflags); + LinuxFile::init(file, fd, buffer, File::DEFAULT_BUFFER_SIZE, _IOFBF, true, + modeflags); return file; } -// TODO: Use the appropriate buffering modes for the standard streams below -// the different buffering modes are available. constexpr size_t STDOUT_BUFFER_SIZE = 1024; char stdout_buffer[STDOUT_BUFFER_SIZE]; -static LinuxFile StdOut(1, stdout_buffer, STDOUT_BUFFER_SIZE, 0, false, +static LinuxFile StdOut(1, stdout_buffer, STDOUT_BUFFER_SIZE, _IOLBF, false, File::ModeFlags(File::OpenMode::APPEND)); File *stdout = &StdOut; -constexpr size_t STDERR_BUFFER_SIZE = 1024; -char stderr_buffer[STDERR_BUFFER_SIZE]; -static LinuxFile StdErr(2, stderr_buffer, STDERR_BUFFER_SIZE, 0, false, +constexpr size_t STDERR_BUFFER_SIZE = 0; +static LinuxFile StdErr(2, nullptr, STDERR_BUFFER_SIZE, _IONBF, false, File::ModeFlags(File::OpenMode::APPEND)); File *stderr = &StdErr; diff --git a/libc/test/src/__support/File/file_test.cpp b/libc/test/src/__support/File/file_test.cpp --- a/libc/test/src/__support/File/file_test.cpp +++ b/libc/test/src/__support/File/file_test.cpp @@ -116,7 +116,8 @@ const char data[] = "hello, file"; constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2; char file_buffer[FILE_BUFFER_SIZE]; - StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "w"); + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "w"); ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); EXPECT_EQ(f->get_pos(), size_t(0)); // Data is buffered in the file stream @@ -133,6 +134,9 @@ EXPECT_GE(f->get_pos(), size_t(0)); ASSERT_EQ(f->flush(), 0); EXPECT_EQ(f->get_pos(), 2 * sizeof(data)); + MemoryView src1("hello, file\0hello, file", sizeof(data) * 2), + dst1(f->get_str(), sizeof(data) * 2); + EXPECT_MEM_EQ(src1, dst1); char read_data[sizeof(data)]; // This is not a readable file. @@ -144,11 +148,58 @@ ASSERT_EQ(f->close(), 0); } +TEST(LlvmLibcFileTest, WriteLineBuffered) { + const char data[] = "hello\n file"; + constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2; + char file_buffer[FILE_BUFFER_SIZE]; + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOLBF, false, "w"); + + ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); + EXPECT_EQ(f->get_pos(), size_t(6)); // buffer after the newline + MemoryView src1("hello\n", 6), dst1(f->get_str(), 6); + EXPECT_MEM_EQ(src1, dst1); + + const char data2[] = "longer for an \n overflow"; + + ASSERT_EQ(sizeof(data2), f->write(data2, sizeof(data2))); + // the buffer's initial contents should be " file\0" + // Writing data2 should cause two flushes, the first being when the buffer + // fills, flushing " file\0longer for a", then the second being when a newline + // is hit, flushing "n \n" and leaving " overflow\0" in the buffer. + EXPECT_EQ(f->get_pos(), size_t(27)); + MemoryView src2("hello\n file\0longer for an \n", 27), dst2(f->get_str(), 27); + EXPECT_MEM_EQ(src2, dst2); + ASSERT_EQ(f->flush(), 0); + EXPECT_EQ(f->get_pos(), sizeof(data) + sizeof(data2)); + MemoryView src3("hello\n file\0longer for an \n overflow", 37), + dst3(f->get_str(), 37); + EXPECT_MEM_EQ(src3, dst3); + + ASSERT_EQ(f->close(), 0); +} + +TEST(LlvmLibcFileTest, WriteUnbuffered) { + const char data[] = "written immediately"; + constexpr size_t FILE_BUFFER_SIZE = sizeof(data) + 1; + char file_buffer[FILE_BUFFER_SIZE]; + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IONBF, false, "w"); + + ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); + EXPECT_EQ(f->get_pos(), + sizeof(data)); // no buffering means this is written immediately. + EXPECT_STREQ(f->get_str(), data); + + ASSERT_EQ(f->close(), 0); +} + TEST(LlvmLibcFileTest, ReadOnly) { const char initial_content[] = "1234567890987654321"; constexpr size_t FILE_BUFFER_SIZE = sizeof(initial_content); char file_buffer[FILE_BUFFER_SIZE]; - StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "r"); + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "r"); f->reset_and_fill(initial_content, sizeof(initial_content)); constexpr size_t READ_SIZE = sizeof(initial_content) / 2; @@ -198,7 +249,8 @@ const char initial_content[] = "1234567890987654321"; constexpr size_t FILE_BUFFER_SIZE = sizeof(initial_content); char file_buffer[FILE_BUFFER_SIZE]; - StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "r"); + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "r"); f->reset_and_fill(initial_content, sizeof(initial_content)); constexpr size_t READ_SIZE = 5; @@ -220,7 +272,8 @@ const char write_data[] = "append"; constexpr size_t FILE_BUFFER_SIZE = sizeof(write_data) * 3 / 2; char file_buffer[FILE_BUFFER_SIZE]; - StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "a"); + StringFile *f = + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "a"); f->reset_and_fill(initial_content, sizeof(initial_content)); constexpr size_t READ_SIZE = 5; @@ -246,7 +299,7 @@ constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2; char file_buffer[FILE_BUFFER_SIZE]; StringFile *f = - new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "w+"); + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "w+"); ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); EXPECT_EQ(f->get_pos(), size_t(0)); // Data is buffered in the file stream @@ -266,7 +319,7 @@ constexpr size_t FILE_BUFFER_SIZE = sizeof(initial_content); char file_buffer[FILE_BUFFER_SIZE]; StringFile *f = - new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "r+"); + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "r+"); f->reset_and_fill(initial_content, sizeof(initial_content)); constexpr size_t READ_SIZE = sizeof(initial_content) / 2; @@ -300,7 +353,7 @@ constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2; char file_buffer[FILE_BUFFER_SIZE]; StringFile *f = - new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "a+"); + new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "a+"); f->reset_and_fill(initial_content, sizeof(initial_content)); ASSERT_EQ(sizeof(data), f->write(data, sizeof(data))); @@ -335,3 +388,33 @@ ASSERT_EQ(f->close(), 0); } + +TEST(LlvmLibcFileTest, SmallBuffer) { + const char WRITE_DATA[] = "small buffer"; + constexpr size_t WRITE_SIZE = sizeof(WRITE_DATA); + constexpr size_t FILE_BUFFER_SIZE = sizeof(WRITE_DATA) / 2 - 1; + char file_buffer[FILE_BUFFER_SIZE]; + StringFile *f = new_string_file(file_buffer, FILE_BUFFER_SIZE, 0, false, "w"); + + ASSERT_EQ(WRITE_SIZE, f->write(WRITE_DATA, WRITE_SIZE)); + // Since data much larger than the buffer is being written, all of it should + // be available in the file without a flush operation. + EXPECT_EQ(f->get_pos(), sizeof(WRITE_DATA)); + ASSERT_STREQ(f->get_str(), WRITE_DATA); + + ASSERT_EQ(f->close(), 0); +} + +TEST(LlvmLibcFileTest, ZeroLengthBuffer) { + const char WRITE_DATA[] = "small buffer"; + constexpr size_t WRITE_SIZE = sizeof(WRITE_DATA); + StringFile *f = new_string_file(nullptr, 0, 0, true, "w"); + + ASSERT_EQ(WRITE_SIZE, f->write(WRITE_DATA, WRITE_SIZE)); + // Since there is no buffer space, all of the written data should + // be available in the file without a flush operation. + EXPECT_EQ(f->get_pos(), sizeof(WRITE_DATA)); + ASSERT_STREQ(f->get_str(), WRITE_DATA); + + ASSERT_EQ(f->close(), 0); +}