Make CertStore natively use File interface (#6131)

__This is a breaking change, but the header and example did warn
everyone that this API was in flux due to the incompatible SD and SPIFFS
File implementations.__

BearSSL CertStores now simply need a filesystem and the names of the
data (generated on-chip) and archive (uploaded by user) files on it.
No more need to roll your own virtual CertStoreFile class.

Update the library, examples, and device test.

Author: earlephilhower

libraries/ESP8266WiFi/examples/BearSSL_CertStore/BearSSL_CertStore.ino

@@ -36,6 +36,7 @@
 #include <ESP8266WiFi.h>
 #include <CertStoreBearSSL.h>
 #include <time.h>
+#include <FS.h>
 
 #ifndef STASSID
 #define STASSID "your-ssid"
@@ -50,87 +51,6 @@ const char *pass = STAPSK;
 // the WiFiClientBearSSLs are present.
 BearSSL::CertStore certStore;
 
-// Uncomment below to use the SD card to store the certs
-// #define USE_SDCARD 1
-
-// NOTE: The CertStoreFile virtual class may migrate to a templated
-// model in a future release. Expect some changes to the interface,
-// no matter what, as the SD and SPIFFS filesystem get unified.
-
-#ifdef USE_SDCARD
-
-#include <SD.h>
-class SDCertStoreFile : public BearSSL::CertStoreFile {
-  public:
-    SDCertStoreFile(const char *name) {
-      _name = name;
-    };
-    virtual ~SDCertStoreFile() override {};
-
-    // The main API
-    virtual bool open(bool write = false) override {
-      _file = SD.open(_name, write ? FILE_WRITE : FILE_READ);
-      return _file;
-    }
-    virtual bool seek(size_t absolute_pos) override {
-      return _file.seek(absolute_pos);
-    }
-    virtual ssize_t read(void *dest, size_t bytes) override {
-      return _file.read(dest, bytes);
-    }
-    virtual ssize_t write(void *dest, size_t bytes) override {
-      return _file.write((const uint8_t*)dest, bytes);
-    }
-    virtual void close() override {
-      _file.close();
-    }
-
-  private:
-    File _file;
-    const char *_name;
-};
-
-SDCertStoreFile certs_idx("/certs.idx"); // Generated by the ESP8266
-SDCertStoreFile certs_ar("/certs.ar"); // Uploaded by the user
-
-#else
-
-#include <FS.h>
-class SPIFFSCertStoreFile : public BearSSL::CertStoreFile {
-  public:
-    SPIFFSCertStoreFile(const char *name) {
-      _name = name;
-    };
-    virtual ~SPIFFSCertStoreFile() override {};
-
-    // The main API
-    virtual bool open(bool write = false) override {
-      _file = SPIFFS.open(_name, write ? "w" : "r");
-      return _file;
-    }
-    virtual bool seek(size_t absolute_pos) override {
-      return _file.seek(absolute_pos, SeekSet);
-    }
-    virtual ssize_t read(void *dest, size_t bytes) override {
-      return _file.readBytes((char*)dest, bytes);
-    }
-    virtual ssize_t write(void *dest, size_t bytes) override {
-      return _file.write((uint8_t*)dest, bytes);
-    }
-    virtual void close() override {
-      _file.close();
-    }
-
-  private:
-    File _file;
-    const char *_name;
-};
-
-SPIFFSCertStoreFile certs_idx("/certs.idx"); // Generated by the ESP8266
-SPIFFSCertStoreFile certs_ar("/certs.ar"); // Uploaded by the user
-
-#endif
-
 // Set time via NTP, as required for x.509 validation
 void setClock() {
   configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov");
@@ -197,11 +117,8 @@ void setup() {
   Serial.println();
   Serial.println();
 
-#ifdef USE_SDCARD
-  SD.begin();
-#else
   SPIFFS.begin();
-#endif
+  // If using a SD card or LittleFS, call the appropriate ::begin instead
 
   // We start by connecting to a WiFi network
   Serial.print("Connecting to ");
@@ -221,10 +138,10 @@ void setup() {
 
   setClock(); // Required for X.509 validation
 
-  int numCerts = certStore.initCertStore(&certs_idx, &certs_ar);
+  int numCerts = certStore.initCertStore(SPIFFS, PSTR("/certs.idx"), PSTR("/certs.ar"));
   Serial.printf("Number of CA certs read: %d\n", numCerts);
   if (numCerts == 0) {
-    Serial.printf("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?\n");
+    Serial.printf("No certs found. Did you run certs-from-mozilla.py and upload the SPIFFS directory before running?\n");
     return; // Can't connect to anything w/o certs!
   }
 
@@ -242,6 +159,9 @@ void loop() {
   do {
     site = Serial.readString();
   } while (site == "");
+  // Strip newline if present
+  site.replace(String("\r"), emptyString);
+  site.replace(String("\n"), emptyString);
   Serial.printf("https://%s/\n", site.c_str());
 
   BearSSL::WiFiClientSecure *bear = new BearSSL::WiFiClientSecure();

libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp

@@ -37,6 +37,12 @@ extern "C" {
   }
 }
 
+
+CertStore::~CertStore() {
+  free(_indexName);
+  free(_dataName);
+}
+
 CertStore::CertInfo CertStore::_preprocessCert(uint32_t length, uint32_t offset, const void *raw) {
   CertStore::CertInfo ci;
 
@@ -70,62 +76,70 @@ CertStore::CertInfo CertStore::_preprocessCert(uint32_t length, uint32_t offset,
 
 // The certs.ar file is a UNIX ar format file, concatenating all the 
 // individual certificates into a single blob in a space-efficient way.
-int CertStore::initCertStore(CertStoreFile *index, CertStoreFile *data) {
+int CertStore::initCertStore(FS &fs, const char *indexFileName, const char *dataFileName) {
   int count = 0;
   uint32_t offset = 0;
 
-  _index = index;
-  _data = data;
+  _fs = &fs;
 
-  if (!_index || !data) {
+  // No strdup_P, so manually do it
+  _indexName = (char *)malloc(strlen_P(indexFileName) + 1);
+  _dataName = (char *)malloc(strlen_P(dataFileName) + 1);
+  if (!_indexName || !_dataName) {
+    free(_indexName);
+    free(_dataName);
     return 0;
   }
+  memcpy_P(_indexName, indexFileName, strlen_P(indexFileName) + 1);
+  memcpy_P(_dataName, dataFileName, strlen_P(dataFileName) + 1);
 
-  if (!_index->open(true)) {
+  File index = _fs->open(_indexName, "w");
+  if (!index) {
     return 0;
   }
 
-  if (!_data->open(false)) {
-    _index->close();
+  File data = _fs->open(_dataName, "r");
+  if (!data) {
+    index.close();
     return 0;
   }
 
-  char magic[8];
-  if (_data->read(magic, sizeof(magic)) != sizeof(magic) ||
+  uint8_t magic[8];
+  if (data.read(magic, sizeof(magic)) != sizeof(magic) ||
       memcmp(magic, "!<arch>\n", sizeof(magic)) ) {
-    _data->close();
-    _index->close();
+    data.close();
+    index.close();
     return 0;
   }
   offset += sizeof(magic);
 
   while (true) {
-    char fileHeader[60];
+    uint8_t fileHeader[60];
     // 0..15 = filename in ASCII
     // 48...57 = length in decimal ASCII
     uint32_t length;
-    if (data->read(fileHeader, sizeof(fileHeader)) != sizeof(fileHeader)) {
+    if (data.read(fileHeader, sizeof(fileHeader)) != sizeof(fileHeader)) {
       break;
     }
     offset += sizeof(fileHeader);
     fileHeader[58] = 0;
-    if (1 != sscanf(fileHeader + 48, "%d", &length) || !length) {
+    if (1 != sscanf((char *)(fileHeader + 48), "%d", &length) || !length) {
       break;
     }
 
     void *raw = malloc(length);
     if (!raw) {
       break;
     }
-    if (_data->read(raw, length) != (ssize_t)length) {
+    if (data.read((uint8_t *)raw, length) != length) {
       free(raw);
       break;
     }
 
     // If the filename starts with "//" then this is a rename file, skip it
     if (fileHeader[0] != '/' || fileHeader[1] != '/') {
       CertStore::CertInfo ci = _preprocessCert(length, offset, raw);
-      if (_index->write(&ci, sizeof(ci)) != (ssize_t)sizeof(ci)) {
+      if (index.write((uint8_t *)&ci, sizeof(ci)) != (ssize_t)sizeof(ci)) {
         free(raw);
         break;
       }
@@ -135,13 +149,13 @@ int CertStore::initCertStore(CertStoreFile *index, CertStoreFile *data) {
     offset += length;
     free(raw);
     if (offset & 1) {
-      char x;
-      _data->read(&x, 1);
+      uint8_t x;
+      data.read(&x, 1);
       offset++;
     }
   }
-  _data->close();
-  _index->close();
+  data.close();
+  index.close();
   return count;
 }
 
@@ -153,35 +167,37 @@ const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn,
   CertStore *cs = static_cast<CertStore*>(ctx);
   CertStore::CertInfo ci;
 
-  if (!cs || len != sizeof(ci.sha256) || !cs->_index || !cs->_data) {
+  if (!cs || len != sizeof(ci.sha256) || !cs->_indexName || !cs->_dataName || !cs->_fs) {
     return nullptr;
   }
 
-  if (!cs->_index->open(false)) {
+  File index = cs->_fs->open(cs->_indexName, "r");
+  if (!index) {
     return nullptr;
   }
 
-  while (cs->_index->read(&ci, sizeof(ci)) == sizeof(ci)) {
+  while (index.read((uint8_t *)&ci, sizeof(ci)) == sizeof(ci)) {
     if (!memcmp(ci.sha256, hashed_dn, sizeof(ci.sha256))) {
-      cs->_index->close();
+      index.close();
       uint8_t *der = (uint8_t*)malloc(ci.length);
       if (!der) {
         return nullptr;
       }
-      if (!cs->_data->open(false)) {
+      File data = cs->_fs->open(cs->_dataName, "r");
+      if (!data) {
         free(der);
         return nullptr;
       }
-      if (!cs->_data->seek(ci.offset)) {
-        cs->_data->close();
+      if (!data.seek(ci.offset, SeekSet)) {
+        data.close();
         free(der);
         return nullptr;
       }
-      if (cs->_data->read(der, ci.length) != (ssize_t)ci.length) {
+      if (data.read((uint8_t *)der, ci.length) != ci.length) {
         free(der);
         return nullptr;
       }
-      cs->_data->close();
+      data.close();
       cs->_x509 = new X509List(der, ci.length);
       free(der);
       if (!cs->_x509) {
@@ -196,7 +212,7 @@ const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn,
       return ta;
     }
   }
-  cs->_index->close();
+  index.close();
   return nullptr;
 }
 

libraries/ESP8266WiFi/src/CertStoreBearSSL.h

@@ -23,50 +23,29 @@
 #include <Arduino.h>
 #include <BearSSLHelpers.h>
 #include <bearssl/bearssl.h>
+#include <FS.h>
 
 // Base class for the certificate stores, which allow use
 // of a large set of certificates stored on SPIFFS of SD card to
 // be dynamically used when validating a X509 certificate
 
 namespace BearSSL {
 
-// Subclass this and provide virtual functions appropriate for your storage.
-// Required because there are conflicting definitions for a "File" in the
-// Arduino setup, and there is no simple way to work around the minor
-// differences.
-// See the examples for implementations to use in your own code.
-//
-// NOTE: This virtual class may migrate to a templated model in a future
-// release.  Expect some changes to the interface, no matter what, as the
-// SD and SPIFFS filesystem get unified.
-class CertStoreFile {
-  public:
-    CertStoreFile() {};
-    virtual ~CertStoreFile() {};
-
-    // The main API
-    virtual bool open(bool write=false) = 0;
-    virtual bool seek(size_t absolute_pos) = 0;
-    virtual ssize_t read(void *dest, size_t bytes) = 0;
-    virtual ssize_t write(void *dest, size_t bytes) = 0;
-    virtual void close() = 0;
-};
-
-
 class CertStore {
   public:
     CertStore() { };
-    ~CertStore() { };
+    ~CertStore();
 
     // Set the file interface instances, do preprocessing
-    int initCertStore(CertStoreFile *index, CertStoreFile *data);
+    int initCertStore(FS &fs, const char *indexFileName, const char *dataFileName);
 
     // Installs the cert store into the X509 decoder (normally via static function callbacks)
     void installCertStore(br_x509_minimal_context *ctx);
 
   protected:
-    CertStoreFile *_index = nullptr;
-    CertStoreFile *_data = nullptr;
+    FS *_fs = nullptr;
+    char *_indexName = nullptr;
+    char *_dataName = nullptr;
     X509List *_x509 = nullptr;
 
     // These need to be static as they are callbacks from BearSSL C code