diff --git a/common/pcapfileformatoptions.cpp b/common/pcapfileformatoptions.cpp new file mode 100644 index 0000000..f3ceeba --- /dev/null +++ b/common/pcapfileformatoptions.cpp @@ -0,0 +1,751 @@ +/* +Copyright (C) 2011 Srivats P. + +This file is part of "Ostinato" + +This is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ + +#include "pcapfileformat.h" + +#include "pdmlreader.h" +#include "ostprotolib.h" +#include "streambase.h" +#include "hexdump.pb.h" + +#include +#include +#include +#include +#include +#include + +const quint32 kPcapFileMagic = 0xa1b2c3d4; +const quint32 kPcapFileMagicSwapped = 0xd4c3b2a1; +const quint32 kNanoSecondPcapFileMagic = 0xa1b23c4d; +const quint32 kNanoSecondPcapFileMagicSwapped = 0x4d3cb2a1; +const quint16 kPcapFileVersionMajor = 2; +const quint16 kPcapFileVersionMinor = 4; +const quint32 kMaxSnapLen = 65535; +const quint32 kDltEthernet = 1; + +PcapFileFormat pcapFileFormat; + +PcapImportOptionsDialog::PcapImportOptionsDialog(QVariantMap *options) + : QDialog(NULL) +{ + setupUi(this); + setAttribute(Qt::WA_DeleteOnClose); + options_ = options; + + viaPdml->setChecked(options_->value("ViaPdml").toBool()); + // XXX: By default this key is absent - so that pcap import tests + // evaluate to false and hence show minimal diffs. + // However, for the GUI user, this should be enabled by default. + recalculateCksums->setChecked( + options_->value("RecalculateCksums", QVariant(true)) + .toBool()); + doDiff->setChecked(options_->value("DoDiff").toBool()); + + connect(buttonBox, SIGNAL(accepted()), this, SLOT(accept())); +} + +PcapImportOptionsDialog::~PcapImportOptionsDialog() +{ +} + +void PcapImportOptionsDialog::accept() +{ + options_->insert("ViaPdml", viaPdml->isChecked()); + options_->insert("RecalculateCksums", recalculateCksums->isChecked()); + options_->insert("DoDiff", doDiff->isChecked()); + + QDialog::accept(); +} + +PcapFileFormat::PcapFileFormat() +{ + importOptions_.insert("ViaPdml", true); + importOptions_.insert("DoDiff", true); +} + +PcapFileFormat::~PcapFileFormat() +{ +} + +bool PcapFileFormat::open(const QString fileName, + OstProto::StreamConfigList &streams, QString &error) +{ + bool isOk = false; + QFile file(fileName); + QTemporaryFile file2; + quint32 magic; + uchar gzipMagic[2]; + bool nsecResolution = false; + int len; + PcapFileHeader fileHdr; + PcapPacketHeader pktHdr; + OstProto::Stream *prevStream = NULL; + quint64 lastXsec = 0; + int pktCount; + qint64 byteCount = 0; + qint64 byteTotal; + QByteArray pktBuf; + bool tryConvert = true; + + if (!file.open(QIODevice::ReadOnly)) + goto _err_open; + + len = file.peek((char*)gzipMagic, sizeof(gzipMagic)); + if (len < int(sizeof(gzipMagic))) + goto _err_reading_magic; + + if ((gzipMagic[0] == 0x1f) && (gzipMagic[1] == 0x8b)) + { + QProcess gzip; + + emit status("Decompressing..."); + emit target(0); + + if (!file2.open()) + { + error.append("Unable to open temporary file to uncompress .gz\n"); + goto _err_unzip_fail; + } + + qDebug("decompressing to %s", qPrintable(file2.fileName())); + + gzip.setStandardOutputFile(file2.fileName()); + gzip.start(OstProtoLib::gzipPath(), + QStringList() + << "-d" + << "-c" + << fileName); + if (!gzip.waitForStarted(-1)) + { + error.append(QString("Unable to start gzip. Check path in Preferences.\n")); + goto _err_unzip_fail; + } + + if (!gzip.waitForFinished(-1)) + { + error.append(QString("Error running gzip\n")); + goto _err_unzip_fail; + } + + file2.seek(0); + + fd_.setDevice(&file2); + } + else + { + fd_.setDevice(&file); + } + +_retry: + byteTotal = fd_.device()->size() - sizeof(fileHdr); + + emit status("Reading File Header..."); + emit target(0); + + fd_ >> magic; + + qDebug("magic = %08x", magic); + + if (magic == kPcapFileMagic) + { + // Do nothing + } + else if (magic == kNanoSecondPcapFileMagic) + { + nsecResolution = true; + } + else if ((magic == kPcapFileMagicSwapped) + || (magic == kNanoSecondPcapFileMagicSwapped)) + { + // Toggle Byte order + if (fd_.byteOrder() == QDataStream::BigEndian) + fd_.setByteOrder(QDataStream::LittleEndian); + else + fd_.setByteOrder(QDataStream::BigEndian); + + nsecResolution = (magic == kNanoSecondPcapFileMagicSwapped); + } + else // Not a pcap file (could be pcapng or something else) + { + if (tryConvert) + { + // Close and reopen the temp file to be safe + file2.close(); + if (!file2.open()) + { + error.append("Unable to open temporary file to convert to PCAP\n"); + goto _err_convert2pcap; + } + fd_.setDevice(0); // disconnect data stream from file + + if (convertToStandardPcap(fileName, file2.fileName(), error)) + { + fd_.setDevice(&file2); + tryConvert = false; + goto _retry; + } + else + { + error = QString(tr("Unable to convert %1 to standard PCAP format")) + .arg(fileName); + goto _err_convert2pcap; + } + } + else + goto _err_bad_magic; + } + + qDebug("reading filehdr"); + + fd_ >> fileHdr.versionMajor; + fd_ >> fileHdr.versionMinor; + fd_ >> fileHdr.thisZone; + fd_ >> fileHdr.sigfigs; + fd_ >> fileHdr.snapLen; + fd_ >> fileHdr.network; + + qDebug("version check"); + if ((fileHdr.versionMajor != kPcapFileVersionMajor) || + (fileHdr.versionMinor != kPcapFileVersionMinor)) + goto _err_unsupported_version; + +#if 1 + // XXX: we support only Ethernet, for now + if (fileHdr.network != kDltEthernet) + goto _err_unsupported_encap; +#endif + + pktBuf.resize(fileHdr.snapLen); + + // XXX: PDML also needs the PCAP file to cross check packet bytes + // with the PDML data, so we can't do PDML conversion any earlier + // than this + qDebug("pdml check"); + if (importOptions_.value("ViaPdml").toBool()) + { + QProcess tshark; + QTemporaryFile pdmlFile; + PdmlReader reader(&streams, importOptions_); + + if (!pdmlFile.open()) + { + error.append("Unable to open temporary file to create PDML\n"); + goto _non_pdml; + } + + qDebug("generating PDML %s", qPrintable(pdmlFile.fileName())); + emit status("Generating PDML..."); + emit target(0); + + tshark.setStandardOutputFile(pdmlFile.fileName()); + tshark.start(OstProtoLib::tsharkPath(), + QStringList() + << QString("-r%1").arg(fileName) + << "-otcp.desegment_tcp_streams:FALSE" + << "-Tpdml"); + if (!tshark.waitForStarted(-1)) + { + error.append(QString("Unable to start tshark. Check path in preferences.\n")); + goto _non_pdml; + } + + if (!tshark.waitForFinished(-1)) + { + error.append(QString("Error running tshark\n")); + goto _non_pdml; + } + + connect(&reader, SIGNAL(progress(int)), this, SIGNAL(progress(int))); + + emit status("Reading PDML packets..."); + emit target(100); // in percentage + + // pdml reader needs pcap, so pass self + isOk = reader.read(&pdmlFile, this, &stop_); + + if (stop_) + goto _user_cancel; + + if (!isOk) + { + error.append(QString("Error processing PDML (%1, %2): %3\n") + .arg(reader.lineNumber()) + .arg(reader.columnNumber()) + .arg(reader.errorString())); + goto _exit; + } + + if (!importOptions_.value("DoDiff").toBool()) + goto _exit; + + + // !-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-! + // Let's do the diff ... + // !-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-! + + QProcess awk; + QProcess diff; + QTemporaryFile originalTextFile; + QTemporaryFile importedPcapFile; + QTemporaryFile importedTextFile; + QTemporaryFile diffFile; + const QString kAwkFilter = + "/^[^0]/ { " + "printf \" %s \", $1;" + "for (i=4; i %s", + qPrintable(originalTextFile.fileName()), + qPrintable(importedTextFile.fileName()), + qPrintable(diffFile.fileName())); + + emit status("Taking diff..."); + emit target(0); + + diff.setStandardOutputFile(diffFile.fileName()); + diff.start(OstProtoLib::diffPath(), + QStringList() + << "-u" + << "-F^ [1-9]" + << QString("--label=%1 (actual)") + .arg(QFileInfo(fileName).fileName()) + << QString("--label=%1 (imported)") + .arg(QFileInfo(fileName).fileName()) + << originalTextFile.fileName() + << importedTextFile.fileName()); + if (!diff.waitForStarted(-1)) + { + error.append(QString("Unable to start diff. Check path in Preferences.\n") + .arg(diff.exitCode())); + goto _diff_fail; + } + + if (!diff.waitForFinished(-1)) + { + error.append(QString("Error running diff\n")); + goto _diff_fail; + } + + diffFile.close(); + if (diffFile.size()) + { + error.append(tr("

There is a diff between the original and imported streams. See details to review the diff.

Why a diff? See possible reasons.

\n\n\n\n").arg("https://jump.ostinato.org/pcapdiff")); + diffFile.open(); + diffFile.seek(0); + error.append(QString(diffFile.readAll())); + } + + goto _exit; + } + +_non_pdml: + qDebug("pcap resolution: %s", nsecResolution ? "nsec" : "usec"); + emit status("Reading Packets..."); + emit target(100); // in percentage + pktCount = 1; + while (!fd_.atEnd()) + { + OstProto::Stream *stream = streams.add_stream(); + OstProto::Protocol *proto = stream->add_protocol(); + OstProto::HexDump *hexDump = proto->MutableExtension(OstProto::hexDump); + + proto->mutable_protocol_id()->set_id( + OstProto::Protocol::kHexDumpFieldNumber); + + readPacket(pktHdr, pktBuf); + + // validations on inclLen <= origLen && inclLen <= snapLen + Q_ASSERT(pktHdr.inclLen <= fileHdr.snapLen); // TODO: convert to if + + hexDump->set_content(pktBuf.data(), pktHdr.inclLen); + hexDump->set_pad_until_end(false); + + stream->mutable_stream_id()->set_id(pktCount); + stream->mutable_core()->set_is_enabled(true); + stream->mutable_core()->set_frame_len(pktHdr.inclLen+4); // FCS + + stream->mutable_control()->set_num_packets(1); + + // setup packet rate to the timing in pcap (as close as possible) + // use quint64 rather than double to store micro/nano second as + // it has a larger range (~580 years) and therefore better accuracy + const quint64 kXsecsInSec = nsecResolution ? 1e9 : 1e6; + quint64 xsec = (pktHdr.tsSec*kXsecsInSec + pktHdr.tsUsec); + quint64 delta = xsec - lastXsec; + qDebug("pktCount = %d, delta = %llu", pktCount, delta); + + if ((pktCount != 1) && delta) + stream->mutable_control()->set_packets_per_sec(double(kXsecsInSec)/delta); + + if (prevStream) + prevStream->mutable_control()->CopyFrom(stream->control()); + + lastXsec = xsec; + prevStream = stream; + pktCount++; + byteCount += pktHdr.inclLen + sizeof(pktHdr); + emit progress(int(byteCount*100/byteTotal)); // in percentage + if (stop_) + goto _user_cancel; + } + + isOk = true; + goto _exit; + +_user_cancel: + isOk = true; + goto _exit; + +_diff_fail: + goto _exit; + +_err_unsupported_encap: + error = QString(tr("%1 has non-ethernet encapsulation (%2) which is " + "not supported - Sorry!")) + .arg(QFileInfo(fileName).fileName()).arg(fileHdr.network); + goto _exit; + +_err_unsupported_version: + error = QString(tr("%1 is in PCAP version %2.%3 format which is " + "not supported - Sorry!")) + .arg(fileName).arg(fileHdr.versionMajor).arg(fileHdr.versionMinor); + goto _exit; + +_err_bad_magic: + error = QString(tr("%1 is not a valid PCAP file")).arg(fileName); + goto _exit; + +#if 0 +_err_truncated: + error = QString(tr("%1 is too short")).arg(fileName); + goto _exit; +#endif + +_err_unzip_fail: + goto _exit; + +_err_reading_magic: + error = QString(tr("Unable to read magic from %1")).arg(fileName); + goto _exit; + +_err_convert2pcap: + goto _exit; + +_err_open: + error = QString(tr("Unable to open file: %1")).arg(fileName); + goto _exit; + +_exit: + if (!error.isEmpty()) + qDebug("%s", qPrintable(error)); + file.close(); + return isOk; +} + +/*! + Converts a non-PCAP capture file to standard PCAP file format using tshark + + Returns true if conversion was successful, false otherwise. +*/ +bool PcapFileFormat::convertToStandardPcap( + QString fileName, QString outputFileName, QString &error) +{ + qDebug("converting to PCAP %s", qPrintable(outputFileName)); + emit status("Unsupported format. Converting to standard PCAP format..."); + emit target(0); + + QProcess tshark; + tshark.start(OstProtoLib::tsharkPath(), + QStringList() + << QString("-r%1").arg(fileName) + << "-Fnsecpcap" + << QString("-w%1").arg(outputFileName)); + if (!tshark.waitForStarted(-1)) + { + error.append(QString("Unable to start tshark. Check path in preferences.\n")); + return false; + } + + if (!tshark.waitForFinished(-1)) + { + error.append(QString("Error running tshark\n")); + return false; + } + + return true; +} + +/*! + Reads packet meta data into pktHdr and packet content into buf. + + Returns true if packet is read successfully, false otherwise. +*/ +bool PcapFileFormat::readPacket(PcapPacketHeader &pktHdr, QByteArray &pktBuf) +{ + quint32 len; + + // TODO: chk fd_.status() + + // read PcapPacketHeader + fd_ >> pktHdr.tsSec; + fd_ >> pktHdr.tsUsec; + fd_ >> pktHdr.inclLen; + fd_ >> pktHdr.origLen; + + // TODO: chk fd_.status() + + // XXX: should never be required, but we play safe + if (quint32(pktBuf.size()) < pktHdr.inclLen) + pktBuf.resize(pktHdr.inclLen); + + // read Pkt contents + len = fd_.readRawData(pktBuf.data(), pktHdr.inclLen); // TODO: use while? + + Q_ASSERT(len == pktHdr.inclLen); // TODO: remove assert + pktBuf.resize(len); + + return true; +} + +bool PcapFileFormat::save(const OstProto::StreamConfigList streams, + const QString fileName, QString &error) +{ + bool isOk = false; + QFile file(fileName); + PcapFileHeader fileHdr; + PcapPacketHeader pktHdr; + QByteArray pktBuf; + + if (!file.open(QIODevice::WriteOnly)) + goto _err_open; + + fd_.setDevice(&file); + + fileHdr.magicNumber = kNanoSecondPcapFileMagic; + fileHdr.versionMajor = kPcapFileVersionMajor; + fileHdr.versionMinor = kPcapFileVersionMinor; + fileHdr.thisZone = 0; + fileHdr.sigfigs = 0; + fileHdr.snapLen = kMaxSnapLen; + fileHdr.network = kDltEthernet; + + fd_ << fileHdr.magicNumber; + fd_ << fileHdr.versionMajor; + fd_ << fileHdr.versionMinor; + fd_ << fileHdr.thisZone; + fd_ << fileHdr.sigfigs; + fd_ << fileHdr.snapLen; + fd_ << fileHdr.network; + + pktBuf.resize(kMaxSnapLen); + + emit status("Writing Packets..."); + emit target(streams.stream_size()); + + pktHdr.tsSec = 0; + pktHdr.tsUsec = 0; + for (int i = 0; i < streams.stream_size(); i++) + { + StreamBase s; + + s.setId(i); + s.protoDataCopyFrom(streams.stream(i)); + // TODO: expand frameIndex for each stream + s.frameValue((uchar*)pktBuf.data(), pktBuf.size(), 0); + + pktHdr.inclLen = s.frameProtocolLength(0); // FIXME: stream index = 0 + pktHdr.origLen = s.frameLen() - 4; // FCS; FIXME: Hardcoding + + qDebug("savepcap i=%d, incl/orig len = %d/%d", i, + pktHdr.inclLen, pktHdr.origLen); + + if (pktHdr.inclLen > fileHdr.snapLen) + pktHdr.inclLen = fileHdr.snapLen; + + fd_ << pktHdr.tsSec; + fd_ << pktHdr.tsUsec; + fd_ << pktHdr.inclLen; + fd_ << pktHdr.origLen; + fd_.writeRawData(pktBuf.data(), pktHdr.inclLen); + + if (s.packetRate()) + { + quint64 delta = quint64(1e9/s.packetRate()); + pktHdr.tsSec += delta/quint32(1e9); + pktHdr.tsUsec += delta % quint32(1e9); + } + + if (pktHdr.tsUsec >= quint32(1e9)) + { + pktHdr.tsSec++; + pktHdr.tsUsec -= quint32(1e9); + } + + emit progress(i); + } + + file.close(); + + isOk = true; + goto _exit; + +_err_open: + error = QString(tr("Unable to open file: %1")).arg(fileName); + goto _exit; + +_exit: + return isOk; +} + +QDialog* PcapFileFormat::openOptionsDialog() +{ + return new PcapImportOptionsDialog(&importOptions_); +} + +bool PcapFileFormat::isMyFileFormat(const QString /*fileName*/) +{ + // TODO + return true; +} + +bool PcapFileFormat::isMyFileType(const QString fileType) +{ + if (fileType.startsWith("PCAP")) + return true; + else + return false; +} diff --git a/common/pcapfileformatoptions.h b/common/pcapfileformatoptions.h new file mode 100644 index 0000000..5e8a744 --- /dev/null +++ b/common/pcapfileformatoptions.h @@ -0,0 +1,88 @@ +/* +Copyright (C) 2011 Srivats P. + +This file is part of "Ostinato" + +This is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see +*/ +#ifndef _PCAP_FILE_FORMAT_H +#define _PCAP_FILE_FORMAT_H + +#include "streamfileformat.h" +#include "ui_pcapfileimport.h" + +#include +#include + +class PcapImportOptionsDialog: public QDialog, public Ui::PcapFileImport +{ +public: + PcapImportOptionsDialog(QVariantMap *options); + ~PcapImportOptionsDialog(); + +private slots: + void accept(); + +private: + QVariantMap *options_; +}; + +class PdmlReader; +class PcapFileFormat : public StreamFileFormat +{ + friend class PdmlReader; + +public: + PcapFileFormat(); + ~PcapFileFormat(); + + bool open(const QString fileName, + OstProto::StreamConfigList &streams, QString &error); + bool save(const OstProto::StreamConfigList streams, + const QString fileName, QString &error); + + virtual QDialog* openOptionsDialog(); + + bool isMyFileFormat(const QString fileName); + bool isMyFileType(const QString fileType); + +private: + typedef struct { + quint32 magicNumber; /* magic number */ + quint16 versionMajor; /* major version number */ + quint16 versionMinor; /* minor version number */ + qint32 thisZone; /* GMT to local correction */ + quint32 sigfigs; /* accuracy of timestamps */ + quint32 snapLen; /* max length of captured packets, in octets */ + quint32 network; /* data link type */ + } PcapFileHeader; + + typedef struct { + quint32 tsSec; /* timestamp seconds */ + quint32 tsUsec; /* timestamp microseconds */ + quint32 inclLen; /* number of octets of packet saved in file */ + quint32 origLen; /* actual length of packet */ + } PcapPacketHeader; + + bool convertToStandardPcap(QString fileName, QString outputFileName, + QString &error); + bool readPacket(PcapPacketHeader &pktHdr, QByteArray &pktBuf); + + QDataStream fd_; + QVariantMap importOptions_; +}; + +extern PcapFileFormat pcapFileFormat; + +#endif