[libc] Handle allocation failures gracefully in FILE related API.

[sivachandra]

Summary

Few uses of free have not yet been replaced by the custom operator
delete yet. They will be done in a follow up patch.

[michaelrj]

LGTM with one nit

libc/src/__support/File/linux_file.cpp

146

could you delete this comment?

[sivachandra]

Remove a commented out line leftover from a pervious change.

libc/src/__support/File/CMakeLists.txt

 if(NOT (TARGET libc.src.__support.threads.mutex))
   # Not all platforms have a mutex implementation. If mutex is unvailable,
   # we just skip everything about files.
   return()
 endif()
 
 add_object_library(
   file
   SRCS
     file.cpp
   HDRS
     file.h
   DEPENDS
     libc.include.errno
+    libc.src.__support.CPP.new
     libc.src.__support.CPP.span
     libc.src.__support.threads.mutex
     libc.src.__support.error_or
 )
 
 add_object_library(
   dir
   SRCS
add_object_library(
     SRCS
       ${LIBC_TARGET_OS}_file.cpp
     DEPENDS
       .file
       libc.include.errno
       libc.include.fcntl
       libc.include.stdio
       libc.include.sys_syscall
+      libc.src.__support.CPP.new
       libc.src.__support.OSUtil.osutil
       libc.src.errno.errno
       libc.src.__support.error_or
   )
 endif()
 
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}_dir.cpp)
   add_object_library(

libc/src/__support/File/file.h

private:
   CloseFunc *platform_close;
   FlushFunc *platform_flush;
 
   Mutex mutex;
 
   // For files which are readable, we should be able to support one ungetc
   // operation even if |buf| is nullptr. So, in the constructor of File, we
   // set |buf| to point to this buffer character.
-  char ungetc_buf;
+  uint8_t ungetc_buf;
 
-  void *buf;      // Pointer to the stream buffer for buffered streams
+  uint8_t *buf;   // Pointer to the stream buffer for buffered streams
   size_t bufsize; // Size of the buffer pointed to by |buf|.
 
   // Buffering mode to used to buffer.
   int bufmode;
 
   // If own_buf is true, the |buf| is owned by the stream and will be
   // free-ed when close method is called on the stream.
   bool own_buf;
public:
   // We want this constructor to be constexpr so that global file objects
   // like stdout do not require invocation of the constructor which can
   // potentially lead to static initialization order fiasco. Consequently,
   // we will assume that the |buffer| and |buffer_size| argument are
   // meaningful - that is, |buffer| is nullptr if and only if |buffer_size|
   // is zero. This way, we will not have to employ the semantics of
   // the set_buffer method and allocate a buffer.
   constexpr File(WriteFunc *wf, ReadFunc *rf, SeekFunc *sf, CloseFunc *cf,
-                 FlushFunc *ff, void *buffer, size_t buffer_size,
+                 FlushFunc *ff, uint8_t *buffer, size_t buffer_size,
                  int buffer_mode, bool owned, ModeFlags modeflags)
       : platform_write(wf), platform_read(rf), platform_seek(sf),
         platform_close(cf), platform_flush(ff), mutex(false, false, false),
         ungetc_buf(0), buf(buffer), bufsize(buffer_size), bufmode(buffer_mode),
         own_buf(owned), mode(modeflags), pos(0), prev_op(FileOp::NONE),
         read_limit(0), eof(false), err(false) {
     adjust_buf();
   }
 
-  // This function helps initialize the various fields of the File data
-  // structure after a allocating memory for it via a call to malloc.
-  static void init(File *f, WriteFunc *wf, ReadFunc *rf, SeekFunc *sf,
-                   CloseFunc *cf, FlushFunc *ff, void *buffer,
-                   size_t buffer_size, int buffer_mode, bool owned,
-                   ModeFlags modeflags) {
-    Mutex::init(&f->mutex, false, false, false);
-    f->platform_write = wf;
-    f->platform_read = rf;
-    f->platform_seek = sf;
-    f->platform_close = cf;
-    f->platform_flush = ff;
-    f->buf = reinterpret_cast<uint8_t *>(buffer);
-    f->bufsize = buffer_size;
-    f->bufmode = buffer_mode;
-    f->own_buf = owned;
-    f->mode = modeflags;
-
-    f->prev_op = FileOp::NONE;
-    f->read_limit = f->pos = 0;
-    f->eof = f->err = false;
-
-    f->adjust_buf();
-  }
-
   // Buffered write of |len| bytes from |data| without the file lock.
   FileIOResult write_unlocked(const void *data, size_t len);
 
   // Buffered write of |len| bytes from |data| under the file lock.
   FileIOResult write(const void *data, size_t len) {
     FileLock l(this);
     return write_unlocked(data, len);
   }
public:
   // |size| is the size of |buffer|. If |size| is non-zero, but |buffer|
   // is nullptr, then a buffer owned by this file will be allocated.
   // Else, |buffer| will not be owned by this file.
   //
   // Will return zero on success, or an error value on failure. Will fail
   // if:
   //   1. |buffer| is not a nullptr but |size| is zero.
   //   2. |buffer_mode| is not one of _IOLBF, IOFBF or _IONBF.
-  // In both the above cases, error returned in EINVAL.
+  //   3. If an allocation was required but the allocation failed.
+  // For cases 1 and 2, the error returned in EINVAL. For case 3, error returned
+  // is ENOMEM.
   int set_buffer(void *buffer, size_t size, int buffer_mode);
 
   // Closes the file stream and frees up all resources owned by it.
   int close();
 
   void lock() { mutex.lock(); }
   void unlock() { mutex.unlock(); }
 

libc/src/__support/File/file.cpp

 //===--- Implementation of a platform independent file data structure -----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "file.h"
 
+#include "src/__support/CPP/new.h"
 #include "src/__support/CPP/span.h"
 
 #include <errno.h> // For error macros
 #include <stdio.h>
 #include <stdlib.h>
 
 namespace __llvm_libc {
 
int File::set_buffer(void *buffer, size_t size, int buffer_mode) {
   default:
     return EINVAL;
   }
 
   if (buffer == nullptr && size != 0 && buffer_mode != _IONBF) {
     // We exclude the case of buffer_mode == _IONBF in this branch
     // because we don't need to allocate buffer in such a case.
     if (own_buf) {
-      buf = realloc(buf, size);
+      // This is one of the places where use a C allocation functon
+      // as C++ does not have an equivalent of realloc.
+      buf = reinterpret_cast<uint8_t *>(realloc(buf, size));
+      if (buf == nullptr)
+        return ENOMEM;
     } else {
-      buf = malloc(size);
+      AllocChecker ac;
+      buf = new (ac) uint8_t[size];
+      if (!ac)
+        return ENOMEM;
       own_buf = true;
     }
     bufsize = size;
     // TODO: Handle allocation failures.
   } else {
     if (own_buf)
-      free(buf);
+      delete buf;
     if (buffer_mode != _IONBF) {
       buf = static_cast<uint8_t *>(buffer);
       bufsize = size;
     } else {
       // We don't need any buffer.
       buf = nullptr;
       bufsize = 0;
     }

libc/src/__support/File/linux_file.cpp

 //===--- Linux specialization of the File data structure ------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "file.h"
 
+#include "src/__support/CPP/new.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 
 #include <errno.h> // For error macros
 #include <fcntl.h> // For mode_t and other flags to the open syscall
 #include <stdio.h>
-#include <stdlib.h>      // For malloc
 #include <sys/syscall.h> // For syscall numbers
 
 namespace __llvm_libc {
 
 namespace {
 
 FileIOResult write_func(File *, const void *, size_t);
 FileIOResult read_func(File *, void *, size_t);
 ErrorOr<long> seek_func(File *, long, int);
 int close_func(File *);
 int flush_func(File *);
 
 } // anonymous namespace
 
 class LinuxFile : public File {
   int fd;
 
 public:
-  constexpr LinuxFile(int file_descriptor, void *buffer, size_t buffer_size,
+  constexpr LinuxFile(int file_descriptor, uint8_t *buffer, size_t buffer_size,
                       int buffer_mode, bool owned, File::ModeFlags modeflags)
       : File(&write_func, &read_func, &seek_func, &close_func, flush_func,
              buffer, buffer_size, buffer_mode, owned, modeflags),
         fd(file_descriptor) {}
 
-  static void init(LinuxFile *f, int file_descriptor, void *buffer,
-                   size_t buffer_size, int buffer_mode, bool owned,
-                   File::ModeFlags modeflags) {
-    File::init(f, &write_func, &read_func, &seek_func, &close_func, &flush_func,
-               buffer, buffer_size, buffer_mode, owned, modeflags);
-    f->fd = file_descriptor;
-  }
-
   int get_fd() const { return fd; }
 };
 
 namespace {
 
 FileIOResult write_func(File *f, const void *data, size_t size) {
   auto *lf = reinterpret_cast<LinuxFile *>(f);
   int ret = __llvm_libc::syscall_impl(SYS_write, lf->get_fd(), data, size);
#elif defined(SYS_openat)
   int fd = __llvm_libc::syscall_impl(SYS_openat, AT_FDCWD, path, open_flags,
                                      OPEN_MODE);
 #else
 #error "SYS_open and SYS_openat syscalls not available to perform a file open."
 #endif
 
   if (fd < 0) {
     return Error(-fd);
     // return {nullptr, -fd};
   }
 
-  void *buffer = malloc(File::DEFAULT_BUFFER_SIZE);
-  auto *file = reinterpret_cast<LinuxFile *>(malloc(sizeof(LinuxFile)));
-  LinuxFile::init(file, fd, buffer, File::DEFAULT_BUFFER_SIZE, _IOFBF, true,
-                  modeflags);
+  uint8_t *buffer;
+  {
+    AllocChecker ac;
+    buffer = new (ac) uint8_t[File::DEFAULT_BUFFER_SIZE];
+    if (!ac) {
+      return Error(ENOMEM);
+    }
+  }
+  AllocChecker ac;
+  auto *file = new (ac)
+      LinuxFile(fd, buffer, File::DEFAULT_BUFFER_SIZE, _IOFBF, true, modeflags);
+  if (!ac)
+    return Error(ENOMEM);
   return file;
 }
 
 int get_fileno(File *f) {
   auto *lf = reinterpret_cast<LinuxFile *>(f);
   return lf->get_fd();
 }
 
 constexpr size_t STDIN_BUFFER_SIZE = 512;
-char stdin_buffer[STDIN_BUFFER_SIZE];
+uint8_t stdin_buffer[STDIN_BUFFER_SIZE];
 static LinuxFile StdIn(0, stdin_buffer, STDIN_BUFFER_SIZE, _IOFBF, false,
                        File::ModeFlags(File::OpenMode::READ));
 File *stdin = &StdIn;
 
 constexpr size_t STDOUT_BUFFER_SIZE = 1024;
-char stdout_buffer[STDOUT_BUFFER_SIZE];
+uint8_t stdout_buffer[STDOUT_BUFFER_SIZE];
 static LinuxFile StdOut(1, stdout_buffer, STDOUT_BUFFER_SIZE, _IOLBF, false,
                         File::ModeFlags(File::OpenMode::APPEND));
 File *stdout = &StdOut;
 
 constexpr size_t STDERR_BUFFER_SIZE = 0;
 static LinuxFile StdErr(2, nullptr, STDERR_BUFFER_SIZE, _IONBF, false,
                         File::ModeFlags(File::OpenMode::APPEND));
 File *stderr = &StdErr;
 
 } // namespace __llvm_libc

libc/src/stdio/CMakeLists.txt

 add_entrypoint_object(
   fopencookie
   SRCS
     fopencookie.cpp
   HDRS
     fopencookie.h
   DEPENDS
     libc.include.stdio
+    libc.src.__support.CPP.new
     libc.src.__support.File.file
 )
 
 add_entrypoint_object(
   stdin
   SRCS
     stdin.cpp
   HDRS

libc/src/stdio/fopencookie.cpp

 //===-- Implementation of fopencookie -------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fopencookie.h"
+#include "src/__support/CPP/new.h"
 #include "src/__support/File/file.h"
 
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 
 namespace __llvm_libc {
 
 namespace {
 
 class CookieFile : public __llvm_libc::File {
-public:
   void *cookie;
   cookie_io_functions_t ops;
+
+  static FileIOResult cookie_write(File *f, const void *data, size_t size);
+  static FileIOResult cookie_read(File *f, void *data, size_t size);
+  static ErrorOr<long> cookie_seek(File *f, long offset, int whence);
+  static int cookie_close(File *f);
+  static int cookie_flush(File *);
+
+public:
+  CookieFile(void *c, cookie_io_functions_t cops, uint8_t *buffer,
+             size_t bufsize, File::ModeFlags mode)
+      : File(&cookie_write, &cookie_read, &CookieFile::cookie_seek,
+             &cookie_close, &cookie_flush, buffer, bufsize,
+             0 /* default buffering mode */, true /* File owns buffer */, mode),
+        cookie(c), ops(cops) {}
 };
 
-FileIOResult write_func(File *f, const void *data, size_t size) {
+FileIOResult CookieFile::cookie_write(File *f, const void *data, size_t size) {
   auto cookie_file = reinterpret_cast<CookieFile *>(f);
   if (cookie_file->ops.write == nullptr)
     return 0;
   return cookie_file->ops.write(cookie_file->cookie,
                                 reinterpret_cast<const char *>(data), size);
 }
 
-FileIOResult read_func(File *f, void *data, size_t size) {
+FileIOResult CookieFile::cookie_read(File *f, void *data, size_t size) {
   auto cookie_file = reinterpret_cast<CookieFile *>(f);
   if (cookie_file->ops.read == nullptr)
     return 0;
   return cookie_file->ops.read(cookie_file->cookie,
                                reinterpret_cast<char *>(data), size);
 }
 
-ErrorOr<long> seek_func(File *f, long offset, int whence) {
+ErrorOr<long> CookieFile::cookie_seek(File *f, long offset, int whence) {
   auto cookie_file = reinterpret_cast<CookieFile *>(f);
   if (cookie_file->ops.seek == nullptr) {
     return Error(EINVAL);
   }
   off64_t offset64 = offset;
   int result = cookie_file->ops.seek(cookie_file->cookie, &offset64, whence);
   if (result == 0)
     return offset64;
   else
     return -1;
 }
 
-int close_func(File *f) {
+int CookieFile::cookie_close(File *f) {
   auto cookie_file = reinterpret_cast<CookieFile *>(f);
   if (cookie_file->ops.close == nullptr)
     return 0;
   return cookie_file->ops.close(cookie_file->cookie);
 }
 
-int flush_func(File *) { return 0; }
+int CookieFile::cookie_flush(File *) { return 0; }
 
 } // anonymous namespace
 
 LLVM_LIBC_FUNCTION(::FILE *, fopencookie,
                    (void *cookie, const char *mode,
                     cookie_io_functions_t ops)) {
-  auto modeflags = File::mode_flags(mode);
-  void *buffer = malloc(File::DEFAULT_BUFFER_SIZE);
-  auto *file = reinterpret_cast<CookieFile *>(malloc(sizeof(CookieFile)));
-  if (file == nullptr)
+  uint8_t *buffer;
+  {
+    AllocChecker ac;
+    buffer = new (ac) uint8_t[File::DEFAULT_BUFFER_SIZE];
+    if (!ac)
+      return nullptr;
+  }
+  AllocChecker ac;
+  auto *file = new (ac) CookieFile(
+      cookie, ops, buffer, File::DEFAULT_BUFFER_SIZE, File::mode_flags(mode));
+  if (!ac)
     return nullptr;
-
-  File::init(file, &write_func, &read_func, &seek_func, &close_func,
-             &flush_func, buffer, File::DEFAULT_BUFFER_SIZE,
-             0,    // Default buffering style
-             true, // Owned buffer
-             modeflags);
-  file->cookie = cookie;
-  file->ops = ops;
-
   return reinterpret_cast<::FILE *>(file);
 }
 
 } // namespace __llvm_libc

libc/test/src/__support/File/CMakeLists.txt

add_libc_unittest(
   SRCS
     file_test.cpp
   LINK_LIBRARIES
     LibcMemoryHelpers
   DEPENDS
     libc.include.errno
     libc.include.stdio
     libc.include.stdlib
+    libc.src.__support.CPP.new
     libc.src.__support.File.file
 )
 
 if (TARGET libc.src.__support.File.platform_file)
   add_libc_unittest(
     platform_file_test
     SUITE
       libc_support_unittests

libc/test/src/__support/File/file_test.cpp

 //===-- Unittests for platform independent file class ---------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
+#include "src/__support/CPP/new.h"
 #include "src/__support/File/file.h"
 #include "src/__support/error_or.h"
 #include "utils/UnitTest/MemoryMatcher.h"
 #include "utils/UnitTest/Test.h"
 
 #include <stdio.h>
 #include <stdlib.h>
 
class StringFile : public __llvm_libc::File {
   static ErrorOr<long> str_seek(__llvm_libc::File *f, long offset, int whence);
   static int str_close(__llvm_libc::File *f) { return 0; }
   static int str_flush(__llvm_libc::File *f) { return 0; }
 
 public:
   explicit StringFile(char *buffer, size_t buflen, int bufmode, bool owned,
                       ModeFlags modeflags)
       : __llvm_libc::File(&str_write, &str_read, &str_seek, &str_close,
-                          &str_flush, buffer, buflen, bufmode, owned,
-                          modeflags),
+                          &str_flush, reinterpret_cast<uint8_t *>(buffer),
+                          buflen, bufmode, owned, modeflags),
         pos(0), eof_marker(0), write_append(false) {
     if (modeflags & static_cast<ModeFlags>(__llvm_libc::File::OpenMode::APPEND))
       write_append = true;
   }
 
-  void init(char *buffer, size_t buflen, int bufmode, bool owned,
-            ModeFlags modeflags) {
-    File::init(this, &str_write, &str_read, &str_seek, &str_close, &str_flush,
-               buffer, buflen, bufmode, owned, modeflags);
-    pos = eof_marker = 0;
-    if (modeflags & static_cast<ModeFlags>(__llvm_libc::File::OpenMode::APPEND))
-      write_append = true;
-    else
-      write_append = false;
-  }
-
   void reset() { pos = 0; }
   size_t get_pos() const { return pos; }
   char *get_str() { return str; }
 
   // Use this method to prefill the file.
   void reset_and_fill(const char *data, size_t len) {
     size_t i;
     for (i = 0; i < len && i < SIZE; ++i) {
if (whence == SEEK_CUR)
     sf->pos += offset;
   if (whence == SEEK_END)
     sf->pos = SIZE + offset;
   return sf->pos;
 }
 
 StringFile *new_string_file(char *buffer, size_t buflen, int bufmode,
                             bool owned, const char *mode) {
-  StringFile *f = reinterpret_cast<StringFile *>(malloc(sizeof(StringFile)));
-  f->init(buffer, buflen, bufmode, owned, __llvm_libc::File::mode_flags(mode));
-  return f;
+  __llvm_libc::AllocChecker ac;
+  // We will just assume the allocation succeeds. We cannot test anything
+  // otherwise.
+  return new (ac) StringFile(buffer, buflen, bufmode, owned,
+                             __llvm_libc::File::mode_flags(mode));
 }
 
 TEST(LlvmLibcFileTest, WriteOnly) {
   const char data[] = "hello, file";
   constexpr size_t FILE_BUFFER_SIZE = sizeof(data) * 3 / 2;
   char file_buffer[FILE_BUFFER_SIZE];
   StringFile *f =
       new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "w");