Summary: How it works: * GetUpdatesSince takes a SequenceNumber. * A LogFile with the first SequenceNumber nearest and lesser than the requested Sequence Number is found. * Seek in the logFile till the requested SeqNumber is found. * Return an iterator which contains logic to return record's one by one. Test Plan: * Test case included to check the good code path. * Will update with more test-cases. * Feedback required on test-cases. Reviewers: dhruba, emayanke Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D7119main
parent
f69e9f3e04
commit
8055008909
@ -0,0 +1,48 @@ |
|||||||
|
// Copyright 2008-present Facebook. All Rights Reserved.
|
||||||
|
|
||||||
|
#ifndef STORAGE_LEVELDB_DB_LOG_FILE_H_ |
||||||
|
#define STORAGE_LEVELDB_DB_LOG_FILE_H_ |
||||||
|
|
||||||
|
namespace leveldb { |
||||||
|
|
||||||
|
enum WalFileType { |
||||||
|
kArchivedLogFile = 0, |
||||||
|
kAliveLogFile = 1 |
||||||
|
} ; |
||||||
|
|
||||||
|
class LogFile { |
||||||
|
|
||||||
|
public: |
||||||
|
uint64_t logNumber; |
||||||
|
WalFileType type; |
||||||
|
|
||||||
|
LogFile(uint64_t logNum,WalFileType logType) : |
||||||
|
logNumber(logNum), |
||||||
|
type(logType) {} |
||||||
|
|
||||||
|
LogFile(const LogFile& that) { |
||||||
|
logNumber = that.logNumber; |
||||||
|
type = that.type; |
||||||
|
} |
||||||
|
|
||||||
|
bool operator < (const LogFile& that) const { |
||||||
|
return logNumber < that.logNumber; |
||||||
|
} |
||||||
|
|
||||||
|
std::string ToString() const { |
||||||
|
char response[100]; |
||||||
|
const char* typeOfLog; |
||||||
|
if (type == kAliveLogFile) { |
||||||
|
typeOfLog = "Alive Log"; |
||||||
|
} else { |
||||||
|
typeOfLog = "Archived Log"; |
||||||
|
} |
||||||
|
sprintf(response, |
||||||
|
"LogNumber : %ld LogType : %s", |
||||||
|
logNumber, |
||||||
|
typeOfLog); |
||||||
|
return std::string(response); |
||||||
|
} |
||||||
|
}; |
||||||
|
} // namespace leveldb
|
||||||
|
#endif // STORAGE_LEVELDB_DB_LOG_FILE_H_
|
@ -0,0 +1,146 @@ |
|||||||
|
#include "db/transaction_log_iterator_impl.h" |
||||||
|
#include "db/write_batch_internal.h" |
||||||
|
#include "db/filename.h" |
||||||
|
namespace leveldb { |
||||||
|
|
||||||
|
TransactionLogIteratorImpl::TransactionLogIteratorImpl( |
||||||
|
const std::string& dbname, |
||||||
|
const Options* options, |
||||||
|
SequenceNumber& seq, |
||||||
|
std::vector<LogFile>* files) : |
||||||
|
dbname_(dbname), |
||||||
|
options_(options), |
||||||
|
sequenceNumber_(seq), |
||||||
|
files_(files), |
||||||
|
started_(false), |
||||||
|
isValid_(true), |
||||||
|
currentFileIndex_(0), |
||||||
|
currentLogReader_(NULL) { |
||||||
|
assert( files_ != NULL); |
||||||
|
} |
||||||
|
|
||||||
|
LogReporter |
||||||
|
TransactionLogIteratorImpl::NewLogReporter(const uint64_t logNumber) { |
||||||
|
LogReporter reporter; |
||||||
|
reporter.env = options_->env; |
||||||
|
reporter.info_log = options_->info_log; |
||||||
|
reporter.log_number = logNumber; |
||||||
|
return reporter; |
||||||
|
} |
||||||
|
|
||||||
|
Status TransactionLogIteratorImpl::OpenLogFile(const LogFile& logFile, |
||||||
|
SequentialFile** file) { |
||||||
|
Env* env = options_->env; |
||||||
|
if (logFile.type == kArchivedLogFile) { |
||||||
|
std::string fname = ArchivedLogFileName(dbname_, logFile.logNumber); |
||||||
|
return env->NewSequentialFile(fname, file); |
||||||
|
} else { |
||||||
|
std::string fname = LogFileName(dbname_, logFile.logNumber); |
||||||
|
Status status = env->NewSequentialFile(fname, file); |
||||||
|
if (!status.ok()) { |
||||||
|
// If cannot open file in DB directory.
|
||||||
|
// Try the archive dir, as it could have moved in the meanwhile.
|
||||||
|
fname = ArchivedLogFileName(dbname_, logFile.logNumber); |
||||||
|
status = env->NewSequentialFile(fname, file); |
||||||
|
if (!status.ok()) { |
||||||
|
// TODO stringprintf
|
||||||
|
return Status::IOError(" Requested file not present in the dir"); |
||||||
|
} |
||||||
|
} |
||||||
|
return status; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void TransactionLogIteratorImpl::GetBatch(WriteBatch* batch) { |
||||||
|
assert(isValid_); // cannot call in a non valid state.
|
||||||
|
WriteBatchInternal::SetContents(batch, currentRecord_); |
||||||
|
} |
||||||
|
|
||||||
|
Status TransactionLogIteratorImpl::status() { |
||||||
|
return currentStatus_; |
||||||
|
} |
||||||
|
|
||||||
|
bool TransactionLogIteratorImpl::Valid() { |
||||||
|
return started_ && isValid_; |
||||||
|
} |
||||||
|
|
||||||
|
void TransactionLogIteratorImpl::Next() { |
||||||
|
// First seek to the given seqNo. in the current file.
|
||||||
|
LogFile currentLogFile = files_->at(currentFileIndex_); |
||||||
|
LogReporter reporter = NewLogReporter(currentLogFile.logNumber); |
||||||
|
std::string scratch; |
||||||
|
Slice record; |
||||||
|
if (!started_) { |
||||||
|
SequentialFile* file = NULL; |
||||||
|
Status status = OpenLogFile(currentLogFile, &file); |
||||||
|
if (!status.ok()) { |
||||||
|
isValid_ = false; |
||||||
|
currentStatus_ = status; |
||||||
|
return; |
||||||
|
} |
||||||
|
assert(file != NULL); |
||||||
|
WriteBatch batch; |
||||||
|
log::Reader* reader = new log::Reader(file, &reporter, true, 0); |
||||||
|
assert(reader != NULL); |
||||||
|
while (reader->ReadRecord(&record, &scratch)) { |
||||||
|
if (record.size() < 12) { |
||||||
|
reporter.Corruption( |
||||||
|
record.size(), Status::Corruption("log record too small")); |
||||||
|
continue; |
||||||
|
} |
||||||
|
WriteBatchInternal::SetContents(&batch, record); |
||||||
|
SequenceNumber currentNum = WriteBatchInternal::Sequence(&batch); |
||||||
|
if (currentNum >= sequenceNumber_) { |
||||||
|
isValid_ = true; |
||||||
|
currentRecord_ = record; |
||||||
|
currentLogReader_ = reader; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
if (!isValid_) { |
||||||
|
// TODO read the entire first file. and did not find the seq number.
|
||||||
|
// Error out.
|
||||||
|
currentStatus_ = |
||||||
|
Status::NotFound("Did not find the Seq no. in first file"); |
||||||
|
} |
||||||
|
started_ = true; |
||||||
|
} else { |
||||||
|
LOOK_NEXT_FILE: |
||||||
|
assert(currentLogReader_ != NULL); |
||||||
|
bool openNextFile = true; |
||||||
|
while (currentLogReader_->ReadRecord(&record, &scratch)) { |
||||||
|
if (record.size() < 12) { |
||||||
|
reporter.Corruption( |
||||||
|
record.size(), Status::Corruption("log record too small")); |
||||||
|
continue; |
||||||
|
} else { |
||||||
|
currentRecord_ = record; |
||||||
|
openNextFile = false; |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (openNextFile) { |
||||||
|
if (currentFileIndex_ < files_->size() - 1) { |
||||||
|
++currentFileIndex_; |
||||||
|
delete currentLogReader_; |
||||||
|
SequentialFile *file; |
||||||
|
Status status = OpenLogFile(files_->at(currentFileIndex_), &file); |
||||||
|
if (!status.ok()) { |
||||||
|
isValid_ = false; |
||||||
|
currentStatus_ = status; |
||||||
|
return; |
||||||
|
} |
||||||
|
currentLogReader_ = new log::Reader(file, &reporter, true, 0); |
||||||
|
goto LOOK_NEXT_FILE; |
||||||
|
} else { |
||||||
|
// LOOKED AT FILES. WE ARE DONE HERE.
|
||||||
|
isValid_ = false; |
||||||
|
currentStatus_ = Status::IOError(" NO MORE DATA LEFT"); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace leveldb
|
@ -0,0 +1,66 @@ |
|||||||
|
// Copyright 2008-present Facebook. All Rights Reserved.
|
||||||
|
#ifndef STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_ |
||||||
|
#define STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_ |
||||||
|
|
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "leveldb/env.h" |
||||||
|
#include "leveldb/options.h" |
||||||
|
#include "leveldb/types.h" |
||||||
|
#include "leveldb/transaction_log_iterator.h" |
||||||
|
#include "db/log_file.h" |
||||||
|
#include "db/log_reader.h" |
||||||
|
|
||||||
|
namespace leveldb { |
||||||
|
|
||||||
|
struct LogReporter : public log::Reader::Reporter { |
||||||
|
Env* env; |
||||||
|
Logger* info_log; |
||||||
|
uint64_t log_number; |
||||||
|
virtual void Corruption(size_t bytes, const Status& s) { |
||||||
|
Log(info_log, "%ld: dropping %d bytes; %s", |
||||||
|
log_number, static_cast<int>(bytes), s.ToString().c_str()); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
class TransactionLogIteratorImpl : public TransactionLogIterator { |
||||||
|
public: |
||||||
|
TransactionLogIteratorImpl(const std::string& dbname, |
||||||
|
const Options* options, |
||||||
|
SequenceNumber& seqNum, |
||||||
|
std::vector<LogFile>* files); |
||||||
|
virtual ~TransactionLogIteratorImpl() { |
||||||
|
// TODO move to cc file.
|
||||||
|
if (currentLogReader_ != NULL) { |
||||||
|
delete currentLogReader_; |
||||||
|
} |
||||||
|
delete files_; |
||||||
|
} |
||||||
|
|
||||||
|
virtual bool Valid(); |
||||||
|
|
||||||
|
virtual void Next(); |
||||||
|
|
||||||
|
virtual Status status(); |
||||||
|
|
||||||
|
virtual void GetBatch(WriteBatch* batch); |
||||||
|
|
||||||
|
private: |
||||||
|
const std::string& dbname_; |
||||||
|
const Options* options_; |
||||||
|
const uint64_t sequenceNumber_; |
||||||
|
const std::vector<LogFile>* files_; |
||||||
|
bool started_; |
||||||
|
bool isValid_; // not valid when it starts of.
|
||||||
|
Status currentStatus_; |
||||||
|
size_t currentFileIndex_; |
||||||
|
Slice currentRecord_; |
||||||
|
log::Reader* currentLogReader_; |
||||||
|
Status OpenLogFile(const LogFile& logFile, SequentialFile** file); |
||||||
|
LogReporter NewLogReporter(uint64_t logNumber); |
||||||
|
}; |
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
} // namespace leveldb
|
||||||
|
#endif // STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
|
@ -0,0 +1,34 @@ |
|||||||
|
// Copyright 2008-present Facebook. All Rights Reserved.
|
||||||
|
#ifndef STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_ |
||||||
|
#define STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_ |
||||||
|
|
||||||
|
#include "leveldb/status.h" |
||||||
|
#include "leveldb/write_batch.h" |
||||||
|
|
||||||
|
namespace leveldb { |
||||||
|
|
||||||
|
|
||||||
|
// A TransactionLogIterator is used to iterate over the Transaction's in a db.
|
||||||
|
class TransactionLogIterator { |
||||||
|
public: |
||||||
|
TransactionLogIterator() {} |
||||||
|
virtual ~TransactionLogIterator() {} |
||||||
|
|
||||||
|
// An iterator is either positioned at a WriteBatch or not valid.
|
||||||
|
// This method returns true if the iterator is valid.
|
||||||
|
virtual bool Valid() = 0; |
||||||
|
|
||||||
|
// Moves the iterator to the next WriteBatch.
|
||||||
|
// REQUIRES: Valid() to be true.
|
||||||
|
virtual void Next() = 0; |
||||||
|
|
||||||
|
// Return's ok if the iterator is in a valid stated.
|
||||||
|
// Return the Error Status when the iterator is not Valid.
|
||||||
|
virtual Status status() = 0; |
||||||
|
|
||||||
|
// If valid return's the current write_batch.
|
||||||
|
virtual void GetBatch(WriteBatch* batch) = 0; |
||||||
|
}; |
||||||
|
} // namespace leveldb
|
||||||
|
|
||||||
|
#endif // STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
|
@ -0,0 +1,14 @@ |
|||||||
|
#ifndef STORAGE_LEVELDB_INCLUDE_TYPES_H_ |
||||||
|
#define STORAGE_LEVELDB_INCLUDE_TYPES_H_ |
||||||
|
|
||||||
|
#include <stdint.h> |
||||||
|
|
||||||
|
namespace leveldb { |
||||||
|
|
||||||
|
// Define all public custom types here.
|
||||||
|
|
||||||
|
// Represents a sequence number in a WAL file.
|
||||||
|
typedef uint64_t SequenceNumber; |
||||||
|
|
||||||
|
} // namespace leveldb
|
||||||
|
#endif // STORAGE_LEVELDB_INCLUDE_TYPES_H_
|
Loading…
Reference in new issue