Summary: How it works: * GetUpdatesSince takes a SequenceNumber. * A LogFile with the first SequenceNumber nearest and lesser than the requested Sequence Number is found. * Seek in the logFile till the requested SeqNumber is found. * Return an iterator which contains logic to return record's one by one. Test Plan: * Test case included to check the good code path. * Will update with more test-cases. * Feedback required on test-cases. Reviewers: dhruba, emayanke Reviewed By: dhruba CC: leveldb Differential Revision: https://reviews.facebook.net/D7119main
parent
f69e9f3e04
commit
8055008909
@ -0,0 +1,48 @@ |
||||
// Copyright 2008-present Facebook. All Rights Reserved.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_FILE_H_ |
||||
#define STORAGE_LEVELDB_DB_LOG_FILE_H_ |
||||
|
||||
namespace leveldb { |
||||
|
||||
enum WalFileType { |
||||
kArchivedLogFile = 0, |
||||
kAliveLogFile = 1 |
||||
} ; |
||||
|
||||
class LogFile { |
||||
|
||||
public: |
||||
uint64_t logNumber; |
||||
WalFileType type; |
||||
|
||||
LogFile(uint64_t logNum,WalFileType logType) : |
||||
logNumber(logNum), |
||||
type(logType) {} |
||||
|
||||
LogFile(const LogFile& that) { |
||||
logNumber = that.logNumber; |
||||
type = that.type; |
||||
} |
||||
|
||||
bool operator < (const LogFile& that) const { |
||||
return logNumber < that.logNumber; |
||||
} |
||||
|
||||
std::string ToString() const { |
||||
char response[100]; |
||||
const char* typeOfLog; |
||||
if (type == kAliveLogFile) { |
||||
typeOfLog = "Alive Log"; |
||||
} else { |
||||
typeOfLog = "Archived Log"; |
||||
} |
||||
sprintf(response, |
||||
"LogNumber : %ld LogType : %s", |
||||
logNumber, |
||||
typeOfLog); |
||||
return std::string(response); |
||||
} |
||||
}; |
||||
} // namespace leveldb
|
||||
#endif // STORAGE_LEVELDB_DB_LOG_FILE_H_
|
@ -0,0 +1,146 @@ |
||||
#include "db/transaction_log_iterator_impl.h" |
||||
#include "db/write_batch_internal.h" |
||||
#include "db/filename.h" |
||||
namespace leveldb { |
||||
|
||||
TransactionLogIteratorImpl::TransactionLogIteratorImpl( |
||||
const std::string& dbname, |
||||
const Options* options, |
||||
SequenceNumber& seq, |
||||
std::vector<LogFile>* files) : |
||||
dbname_(dbname), |
||||
options_(options), |
||||
sequenceNumber_(seq), |
||||
files_(files), |
||||
started_(false), |
||||
isValid_(true), |
||||
currentFileIndex_(0), |
||||
currentLogReader_(NULL) { |
||||
assert( files_ != NULL); |
||||
} |
||||
|
||||
LogReporter |
||||
TransactionLogIteratorImpl::NewLogReporter(const uint64_t logNumber) { |
||||
LogReporter reporter; |
||||
reporter.env = options_->env; |
||||
reporter.info_log = options_->info_log; |
||||
reporter.log_number = logNumber; |
||||
return reporter; |
||||
} |
||||
|
||||
Status TransactionLogIteratorImpl::OpenLogFile(const LogFile& logFile, |
||||
SequentialFile** file) { |
||||
Env* env = options_->env; |
||||
if (logFile.type == kArchivedLogFile) { |
||||
std::string fname = ArchivedLogFileName(dbname_, logFile.logNumber); |
||||
return env->NewSequentialFile(fname, file); |
||||
} else { |
||||
std::string fname = LogFileName(dbname_, logFile.logNumber); |
||||
Status status = env->NewSequentialFile(fname, file); |
||||
if (!status.ok()) { |
||||
// If cannot open file in DB directory.
|
||||
// Try the archive dir, as it could have moved in the meanwhile.
|
||||
fname = ArchivedLogFileName(dbname_, logFile.logNumber); |
||||
status = env->NewSequentialFile(fname, file); |
||||
if (!status.ok()) { |
||||
// TODO stringprintf
|
||||
return Status::IOError(" Requested file not present in the dir"); |
||||
} |
||||
} |
||||
return status; |
||||
} |
||||
} |
||||
|
||||
void TransactionLogIteratorImpl::GetBatch(WriteBatch* batch) { |
||||
assert(isValid_); // cannot call in a non valid state.
|
||||
WriteBatchInternal::SetContents(batch, currentRecord_); |
||||
} |
||||
|
||||
Status TransactionLogIteratorImpl::status() { |
||||
return currentStatus_; |
||||
} |
||||
|
||||
bool TransactionLogIteratorImpl::Valid() { |
||||
return started_ && isValid_; |
||||
} |
||||
|
||||
void TransactionLogIteratorImpl::Next() { |
||||
// First seek to the given seqNo. in the current file.
|
||||
LogFile currentLogFile = files_->at(currentFileIndex_); |
||||
LogReporter reporter = NewLogReporter(currentLogFile.logNumber); |
||||
std::string scratch; |
||||
Slice record; |
||||
if (!started_) { |
||||
SequentialFile* file = NULL; |
||||
Status status = OpenLogFile(currentLogFile, &file); |
||||
if (!status.ok()) { |
||||
isValid_ = false; |
||||
currentStatus_ = status; |
||||
return; |
||||
} |
||||
assert(file != NULL); |
||||
WriteBatch batch; |
||||
log::Reader* reader = new log::Reader(file, &reporter, true, 0); |
||||
assert(reader != NULL); |
||||
while (reader->ReadRecord(&record, &scratch)) { |
||||
if (record.size() < 12) { |
||||
reporter.Corruption( |
||||
record.size(), Status::Corruption("log record too small")); |
||||
continue; |
||||
} |
||||
WriteBatchInternal::SetContents(&batch, record); |
||||
SequenceNumber currentNum = WriteBatchInternal::Sequence(&batch); |
||||
if (currentNum >= sequenceNumber_) { |
||||
isValid_ = true; |
||||
currentRecord_ = record; |
||||
currentLogReader_ = reader; |
||||
break; |
||||
} |
||||
} |
||||
if (!isValid_) { |
||||
// TODO read the entire first file. and did not find the seq number.
|
||||
// Error out.
|
||||
currentStatus_ = |
||||
Status::NotFound("Did not find the Seq no. in first file"); |
||||
} |
||||
started_ = true; |
||||
} else { |
||||
LOOK_NEXT_FILE: |
||||
assert(currentLogReader_ != NULL); |
||||
bool openNextFile = true; |
||||
while (currentLogReader_->ReadRecord(&record, &scratch)) { |
||||
if (record.size() < 12) { |
||||
reporter.Corruption( |
||||
record.size(), Status::Corruption("log record too small")); |
||||
continue; |
||||
} else { |
||||
currentRecord_ = record; |
||||
openNextFile = false; |
||||
break; |
||||
} |
||||
} |
||||
|
||||
if (openNextFile) { |
||||
if (currentFileIndex_ < files_->size() - 1) { |
||||
++currentFileIndex_; |
||||
delete currentLogReader_; |
||||
SequentialFile *file; |
||||
Status status = OpenLogFile(files_->at(currentFileIndex_), &file); |
||||
if (!status.ok()) { |
||||
isValid_ = false; |
||||
currentStatus_ = status; |
||||
return; |
||||
} |
||||
currentLogReader_ = new log::Reader(file, &reporter, true, 0); |
||||
goto LOOK_NEXT_FILE; |
||||
} else { |
||||
// LOOKED AT FILES. WE ARE DONE HERE.
|
||||
isValid_ = false; |
||||
currentStatus_ = Status::IOError(" NO MORE DATA LEFT"); |
||||
} |
||||
} |
||||
|
||||
} |
||||
} |
||||
|
||||
} // namespace leveldb
|
@ -0,0 +1,66 @@ |
||||
// Copyright 2008-present Facebook. All Rights Reserved.
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_ |
||||
#define STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_ |
||||
|
||||
#include <vector> |
||||
|
||||
#include "leveldb/env.h" |
||||
#include "leveldb/options.h" |
||||
#include "leveldb/types.h" |
||||
#include "leveldb/transaction_log_iterator.h" |
||||
#include "db/log_file.h" |
||||
#include "db/log_reader.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
struct LogReporter : public log::Reader::Reporter { |
||||
Env* env; |
||||
Logger* info_log; |
||||
uint64_t log_number; |
||||
virtual void Corruption(size_t bytes, const Status& s) { |
||||
Log(info_log, "%ld: dropping %d bytes; %s", |
||||
log_number, static_cast<int>(bytes), s.ToString().c_str()); |
||||
} |
||||
}; |
||||
|
||||
class TransactionLogIteratorImpl : public TransactionLogIterator { |
||||
public: |
||||
TransactionLogIteratorImpl(const std::string& dbname, |
||||
const Options* options, |
||||
SequenceNumber& seqNum, |
||||
std::vector<LogFile>* files); |
||||
virtual ~TransactionLogIteratorImpl() { |
||||
// TODO move to cc file.
|
||||
if (currentLogReader_ != NULL) { |
||||
delete currentLogReader_; |
||||
} |
||||
delete files_; |
||||
} |
||||
|
||||
virtual bool Valid(); |
||||
|
||||
virtual void Next(); |
||||
|
||||
virtual Status status(); |
||||
|
||||
virtual void GetBatch(WriteBatch* batch); |
||||
|
||||
private: |
||||
const std::string& dbname_; |
||||
const Options* options_; |
||||
const uint64_t sequenceNumber_; |
||||
const std::vector<LogFile>* files_; |
||||
bool started_; |
||||
bool isValid_; // not valid when it starts of.
|
||||
Status currentStatus_; |
||||
size_t currentFileIndex_; |
||||
Slice currentRecord_; |
||||
log::Reader* currentLogReader_; |
||||
Status OpenLogFile(const LogFile& logFile, SequentialFile** file); |
||||
LogReporter NewLogReporter(uint64_t logNumber); |
||||
}; |
||||
|
||||
|
||||
|
||||
} // namespace leveldb
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
|
@ -0,0 +1,34 @@ |
||||
// Copyright 2008-present Facebook. All Rights Reserved.
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_ |
||||
#define STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_ |
||||
|
||||
#include "leveldb/status.h" |
||||
#include "leveldb/write_batch.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
|
||||
// A TransactionLogIterator is used to iterate over the Transaction's in a db.
|
||||
class TransactionLogIterator { |
||||
public: |
||||
TransactionLogIterator() {} |
||||
virtual ~TransactionLogIterator() {} |
||||
|
||||
// An iterator is either positioned at a WriteBatch or not valid.
|
||||
// This method returns true if the iterator is valid.
|
||||
virtual bool Valid() = 0; |
||||
|
||||
// Moves the iterator to the next WriteBatch.
|
||||
// REQUIRES: Valid() to be true.
|
||||
virtual void Next() = 0; |
||||
|
||||
// Return's ok if the iterator is in a valid stated.
|
||||
// Return the Error Status when the iterator is not Valid.
|
||||
virtual Status status() = 0; |
||||
|
||||
// If valid return's the current write_batch.
|
||||
virtual void GetBatch(WriteBatch* batch) = 0; |
||||
}; |
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
|
@ -0,0 +1,14 @@ |
||||
#ifndef STORAGE_LEVELDB_INCLUDE_TYPES_H_ |
||||
#define STORAGE_LEVELDB_INCLUDE_TYPES_H_ |
||||
|
||||
#include <stdint.h> |
||||
|
||||
namespace leveldb { |
||||
|
||||
// Define all public custom types here.
|
||||
|
||||
// Represents a sequence number in a WAL file.
|
||||
typedef uint64_t SequenceNumber; |
||||
|
||||
} // namespace leveldb
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_TYPES_H_
|
Loading…
Reference in new issue