//------------------------------------------------------------------------------ // // Project: Anonymizer // // Brno University of Technology // Faculty of Information Technology // //------------------------------------------------------------------------------ // // This project was financially supported by project VG20102015006 funds // provided by Ministry of the Interior of the Czech republic. // //------------------------------------------------------------------------------ /*! @file videotracker.cpp @brief Implementation of methods ... @details Details @authors Martin Borek (mborekcz@gmail.com) @authors Filip Orsag (orsag@fit.vutbr.cz) @date 2014-2015 @note This project was supported by MV CR project VG20102015006. @copyright BUT OPEN SOURCE LICENCE (see License.txt) */ #include #include #include "videotracker.h" #include "avwriter.h" using namespace cv; VideoTracker::VideoTracker() { player = nullptr; currentFrame = nullptr; tempFrame = nullptr; qDebug() << "new videoTracker"; } VideoTracker::VideoTracker(std::string const &videoAddr, QProgressDialog const *progressDialog) { load_video(videoAddr, progressDialog); } void VideoTracker::load_video(std::string const &videoAddr, QProgressDialog const *progressDialog) { av_register_all(); player = new FFmpegPlayer(videoAddr, progressDialog); currentFrame = new VideoFrame(player->get_width(), player->get_height()); // stores currently read frame tempFrame = new VideoFrame(player->get_width(), player->get_height()); // stores temporary frame when tracking } VideoTracker::~VideoTracker() { /** for (TrackedObject *object: trackedObjects) { delete object; object = nullptr; } */ delete currentFrame; currentFrame = nullptr; delete tempFrame; tempFrame = nullptr; delete player; player = nullptr; } void VideoTracker::seek_first_packet() { if (!player->seek_first_packet()) { qDebug() << "ERROR-create_output: seek_first_packet"; } } int VideoTracker::add_object(Characteristics const &objectAppearance, std::string const &objectName, Selection initialPosition, int64_t initialTimestamp, unsigned long initialTimePosition, unsigned long initialFrameNumber, bool endTimestampSet, int64_t endTimestamp, unsigned long endTimePosition, unsigned long endFrameNumber) { qDebug() << "Initialize tracking"; auto newObject = std::make_shared(objectAppearance, objectName, initialTimestamp, initialPosition, initialTimePosition, initialFrameNumber, endTimestampSet, endTimestamp, endTimePosition, endFrameNumber); trackedObjects.push_back(newObject); if (!player->get_frame_by_timestamp(currentFrame, initialTimestamp)) { qDebug()<< "Tracker: Cannot read this frame."; return -1; } newObject->track_next(currentFrame); qDebug() << "Initialized"; return trackedObjects.size() - 1; } bool VideoTracker::get_frame_by_time(QImage &imgFrame, QImage &originalImgFrame, bool includeOriginal, int64_t time, QProgressDialog *progressDialog) { if (!player->get_frame_by_time(currentFrame, time)) return false; Mat result = currentFrame->get_mat_frame()->clone(); // Simple assignment would do only a shallow copy, clone() is needed if (!track_frame(currentFrame, result, progressDialog)) return false; if (includeOriginal) originalImgFrame = Mat2QImage(*currentFrame->get_mat_frame()); imgFrame = Mat2QImage(result); return true; } bool VideoTracker::get_frame_by_timestamp(QImage &imgFrame, QImage &originalImgFrame, bool includeOriginal, int64_t timestamp, QProgressDialog *progressDialog) { if (!player->get_frame_by_timestamp(currentFrame, timestamp)) return false; Mat result = currentFrame->get_mat_frame()->clone(); // Simple assignment would do only a shallow copy, clone() is needed if (!track_frame(currentFrame, result, progressDialog)) return false; if (includeOriginal) originalImgFrame = Mat2QImage(*currentFrame->get_mat_frame()); imgFrame = Mat2QImage(result); return true; } bool VideoTracker::get_frame_by_number(QImage &imgFrame, QImage &originalImgFrame, bool includeOriginal, unsigned long frameNumber, QProgressDialog *progressDialog) { if (!player->get_frame_by_number(currentFrame, frameNumber)) return false; Mat result = currentFrame->get_mat_frame()->clone(); // Simple assignment would do only a shallow copy, clone() is needed if (!track_frame(currentFrame, result, progressDialog)) return false; if (includeOriginal) originalImgFrame = Mat2QImage(*currentFrame->get_mat_frame()); imgFrame = Mat2QImage(result); return true; } bool VideoTracker::get_next_frame(QImage &imgFrame, QImage &originalImgFrame, bool includeOriginal, QProgressDialog *progressDialog) { //Mat matFrame(player->get_height(), player->get_width(), CV_8UC3); // This variable is used for the "track_frame" function to determine whether // it needs to jump to a particular frame or if the lastProcessedTimestamp is // the same as this variable (previousTimestamp). When it's the same, // track_frame does not need to jump since the currentlly read frame is the // following to the previously tracked. int64_t previousTimestamp = currentFrame->get_timestamp(); if (!player->get_next_frame(currentFrame)) return false; Mat result = currentFrame->get_mat_frame()->clone(); // Simple assignment would do only a shallow copy, clone() is needed if (!track_frame(currentFrame, result, progressDialog, true, previousTimestamp)) return false; if (includeOriginal) originalImgFrame = Mat2QImage(*currentFrame->get_mat_frame()); imgFrame = Mat2QImage(result); return true; } bool VideoTracker::get_previous_frame(QImage &imgFrame, QImage &originalImgFrame, bool includeOriginal, QProgressDialog *progressDialog) { if (!player->get_previous_frame(currentFrame)) return false; Mat result = currentFrame->get_mat_frame()->clone(); // Simple assignment would do only a shallow copy, clone() is needed if (!track_frame(currentFrame, result, progressDialog)) return false; if (includeOriginal) originalImgFrame = Mat2QImage(*currentFrame->get_mat_frame()); imgFrame = Mat2QImage(result); return true; } bool VideoTracker::get_first_frame(QImage &imgFrame, QImage &originalImgFrame, bool includeOriginal, QProgressDialog *progressDialog) { player->seek_first_packet(); return get_next_frame(imgFrame, originalImgFrame, includeOriginal, progressDialog); } bool VideoTracker::get_current_frame(QImage &imgFrame, QImage &originalImgFrame, bool includeOriginal) { Mat result = currentFrame->get_mat_frame()->clone(); // Simple assignment would do only a shallow copy, clone() is needed if (!track_frame(currentFrame, result, nullptr)) return false; if (includeOriginal) originalImgFrame = Mat2QImage(*currentFrame->get_mat_frame()); imgFrame = Mat2QImage(result); return true; } bool VideoTracker::track_frame(VideoFrame const *originalFrame, cv::Mat &result, QProgressDialog *progressDialog, bool previousTimestampSet, int64_t previousTimestamp) { int64_t currentTimestamp = originalFrame->get_timestamp(); if (!trackedObjects.empty()) { Selection trackedPosition; //for (TrackedObject *object: trackedObjects) for (auto object: trackedObjects) { if ((object->get_initial_timestamp() > currentTimestamp) || ((object->is_end_timestamp_set() && object->get_end_timestamp() < currentTimestamp))) continue; // In this case current timestamp is out of the range of this tracked object int64_t lastProcessedTimestamp; bool lastProcessedTimestampSet = object->get_last_processed_timestamp(lastProcessedTimestamp); // If the frame was already processed but the whole video is not yet processed if (object->is_all_processed() || (lastProcessedTimestampSet && currentTimestamp <= lastProcessedTimestamp)) { if (!object->get_position(currentTimestamp, trackedPosition))// Position is already known as this frame has already been processed. { return false; } } else if (previousTimestampSet && (lastProcessedTimestamp == previousTimestamp)) // Is used only with function "get_next_frame"; instead of seeking frame it uses the current one as it is the right one when obtained by get_next_frame(); that's much faster trackedPosition = object->track_next(originalFrame); /**else if (!object->is_initialized()) { qDebug() << "ERROR - VideoTracker: tracking frame when trackedObject not initialized"; return false; }*/ else { if (progressDialog) { progressDialog->show(); } if (lastProcessedTimestampSet) { // Set to the last processed position. This frame won't be used, but allows to use "get_next_frame". // It might be possible to skip this frame and find right the desired (next) one. // However, it would be more complicated and won't make it much faster. if (!player->get_frame_by_timestamp(tempFrame, lastProcessedTimestamp)) return false; if (!player->get_next_frame(tempFrame)) // No more frames => all processed for this object { object->set_all_processed(true); continue; } } else { // So far no frame is processed, therefore lastProcessedTimestamp is not set if (!player->get_frame_by_timestamp(tempFrame, object->get_initial_timestamp())) return false; } while (true) { qApp->processEvents(); // Keeps progress bar active trackedPosition = object->track_next(tempFrame); if (tempFrame->get_timestamp() >= currentTimestamp) break; if (progressDialog) { if (progressDialog->wasCanceled()) // User cancelled the progress dialog throw UserCanceledException(); } qApp->processEvents(); // Keeps progress bar active if (!player->get_next_frame(tempFrame)) return false; // Could not reach desired frame } //while (object->get_last_processed_timestamp() < currentTimestamp); assert(currentTimestamp == tempFrame->get_timestamp()); if (progressDialog) { if (progressDialog->wasCanceled()) // User cancelled the progress dialog throw UserCanceledException(); } } if (!object->draw_mark(result, *originalFrame->get_mat_frame(), currentTimestamp)) return false; } //progressDialog->reset(); return true; } return true; } bool VideoTracker::track_object(unsigned int objectID, QProgressDialog *progressDialog) { auto object = trackedObjects[objectID]; if (object->is_all_processed()) { return true; // This object is processed throughout all its range } else { int64_t endTimestamp = object->get_end_timestamp(); int64_t lastProcessedTimestamp; bool lastProcessedTimestampSet = object->get_last_processed_timestamp(lastProcessedTimestamp); // If the frame was already processed but the whole video is not yet processed if (lastProcessedTimestampSet) { // Set to the last processed position. This frame won't be used, but allows to use "get_next_frame". // It might be possible to skip this frame and find right the desired (next) one. // However, it would be more complicated and won't make it much faster. if (!player->get_frame_by_timestamp(tempFrame, lastProcessedTimestamp)) { //progressDialog->reset(); return false; } if (!player->get_next_frame(tempFrame)) // No more frames => all processed for this object { object->set_all_processed(true); return true; } } else { // So far no frame is processed, therefore lastProcessedTimestamp is not set if (!player->get_frame_by_timestamp(tempFrame, object->get_initial_timestamp())) { //progressDialog->reset(); return false; } } while (true) { qApp->processEvents(); // Keeps progress bar active object->track_next(tempFrame); // Also sets object->allProcessed if (object->is_end_timestamp_set() && (tempFrame->get_timestamp() >= endTimestamp)) break; if (progressDialog->wasCanceled()) // User canceled the progress dialog { // reads last tracked frame; is used for knowing where aborted player->get_current_frame(currentFrame); throw UserCanceledException(); } qApp->processEvents(); // Keeps progress bar active if (!player->get_next_frame(tempFrame)) { if (object->is_end_timestamp_set()) { qDebug() << "Warning-track_all(): Cannot read more frames but object->endTimestamp is higher"; } object->set_all_processed(true); break; } } } return true; } bool VideoTracker::track_all(QProgressDialog *progressDialog) { if (!trackedObjects.empty()) { progressDialog->show(); unsigned int i; for (i = 0; i < trackedObjects.size(); i++) { if (!track_object(i, progressDialog)) { // Tracking is not successful, do not track other objects qDebug() << "ERROR-track_all(): track_object() returned false"; return false; } } progressDialog->cancel(); //progressDialog->reset(); return true; } qDebug() << "Warning-track_all(): No objects to track"; return true; } QImage VideoTracker::Mat2QImage(Mat const &src) const { Mat temp; // make the same cv::Mat cvtColor(src, temp, CV_BGR2RGB); QImage dest((const uchar *) temp.data, temp.cols, temp.rows, temp.step, QImage::Format_RGB888); dest.bits(); // enforce deep copy, see documentation // of QImage::QImage ( const uchar * data, int width, int height, Format format ) return dest; } /** bool VideoTracker::set_frame_position(int const &position) { return player->set_frame_position(position); } bool VideoTracker::set_time_position(int const &position) { return player->set_time_position(position); } */ double VideoTracker::get_fps() const { return player->get_fps(); } unsigned long VideoTracker::get_frame_count() const { return player->get_frame_count(); } unsigned long VideoTracker::get_total_time() const { // return player->get_frame_count() / player->get_fps(); return player->get_total_time(); } int64_t VideoTracker::get_frame_timestamp() const { return currentFrame->get_timestamp(); } unsigned long VideoTracker::get_time_position() const { return currentFrame->get_time_position(); } unsigned long VideoTracker::get_frame_number() const { return currentFrame->get_frame_number(); } unsigned int VideoTracker::get_objects_count() const { return trackedObjects.size(); } void VideoTracker::create_output(std::string const &filename, QProgressDialog &fileProgressDialog, QProgressDialog *trackingProgressDialog, std::string inFileExtension) { qDebug() << "filename" <cancel(); qDebug() << "Creating output: Tracking successful"; qDebug() << "Creating output: Writer initialization"; AVWriter *writer = new AVWriter(); if (!writer->initialize_output(filename, player->get_video_stream(), player->get_audio_stream(), player->get_format_name(), inFileExtension)) { //qDebug() << "Error: Writer not correctly initialized"; throw OutputException(); //return false; } qDebug() << "Creating output: Writer successfully initialized"; AVPacket packet; bool isAudio = false; //int i = 0; qDebug() << "Creating output: Seeking first packet"; if (!player->seek_first_packet()) { qDebug() << "ERROR-create_output: seek_first_packet"; throw OutputException(); //return false; } while (player->read_frame(packet, false/*onlyVideoPackets*/, &isAudio)) { //packet needs to be freed only when contains audio, otherwise is freed already in read_frame() if (isAudio) { if (!writer->write_audio_packet(packet)) { av_free_packet(&packet); throw OutputException(); } else av_free_packet(&packet); } else { if (!player->get_current_frame(currentFrame)) { qDebug() << "Error: creating output: cannot get frame."; throw OutputException(); } // Mat result = currentFrame->get_mat_frame()->clone(); // Simple assignment would do only a shallow copy, clone() is needed VideoFrame result (*currentFrame); // Simple assignment would do only a shallow copy, clone() is needed // All trajectories are counted with track_all(); // track_frame now only returns those previously counted trajectories. if (!track_frame(currentFrame, *(result.matFrame))) break; if (!writer->write_video_frame(result)) { qDebug() << "Error: write_video_frame"; throw OutputException(); } qDebug() << currentFrame->get_time_position() << "/" << get_total_time(); fileProgressDialog.setValue(currentFrame->get_time_position()); if (fileProgressDialog.wasCanceled()) { // todo: close file or delete? writer->close_output(); delete writer; throw UserCanceledException(); } } } if (!writer->write_last_frames()) // There might be frames left in the buffer qDebug() << "WARNING: Creating output: write_last_frames() not correct"; qDebug() << "Creating output: Frames successfully written"; qDebug() << "Creating output: Closing output file"; if (!writer->close_output()) { qDebug() << "Error: Writer not correctly closed"; throw OutputException(); //return false; } // fileProgressDialog.setValue(fileProgressDialog.maximum()); // In case video was shorter than the claimed length delete writer; writer = nullptr; player->seek_first_packet(); // Return video to the first position qDebug() << "Creating output: Output file successfully closed"; //return true; } Characteristics VideoTracker::get_object_appearance(unsigned int objectID) const { return trackedObjects[objectID]->get_appearance(); } std::map const &VideoTracker::get_object_trajectory_sections(unsigned int objectID) const { return trackedObjects[objectID]->get_trajectory_sections(); } std::map const &VideoTracker::get_object_trajectory(unsigned int objectID) const { return trackedObjects[objectID]->get_trajectory(); } bool VideoTracker::set_object_trajectory_section(unsigned int objectID, int64_t newTimestamp, Selection position, unsigned long timePosition, unsigned long frameNumber) { return trackedObjects[objectID]->set_trajectory_section(newTimestamp, position, timePosition, frameNumber); //todo check if exists } bool VideoTracker::change_object_trajectory_section(unsigned int objectID, int64_t oldTimestamp, int64_t newTimestamp, Selection position, unsigned long timePosition, unsigned long frameNumber) { return trackedObjects[objectID]->change_trajectory_section(oldTimestamp, newTimestamp, position, timePosition, frameNumber); //todo check if exists } bool VideoTracker::delete_object_trajectory_section(unsigned int objectID, int64_t timestamp) { return trackedObjects[objectID]->delete_trajectory_section(timestamp); //todo check if exists } bool VideoTracker::change_object_end_frame(unsigned int objectID, bool set, int64_t timestamp, unsigned long timePosition, unsigned long frameNumber) { return trackedObjects[objectID]->change_end_frame(set, timestamp, timePosition, frameNumber); //todo test if object exists } bool VideoTracker::get_object_end(unsigned int objectID, int64_t ×tamp, unsigned long &timePosition, unsigned long &frameNumber) { if (!trackedObjects[objectID]->is_end_timestamp_set()) return false; timestamp = trackedObjects[objectID]->get_end_timestamp(); timePosition = trackedObjects[objectID]->get_end_time_position(); frameNumber = trackedObjects[objectID]->get_end_frame_number(); return true; } bool VideoTracker::change_object_appearance(unsigned int objectID, Characteristics const &newAppearance) { trackedObjects[objectID]->change_appearance(newAppearance); return true; } std::string VideoTracker::get_object_name(unsigned int objectID) { return trackedObjects[objectID]->get_name(); } void VideoTracker::set_object_name(unsigned int objectID, std::string newName) { trackedObjects[objectID]->set_name(newName); } void VideoTracker::delete_object(unsigned int objectID) { assert(trackedObjects.size() > objectID); trackedObjects.erase(trackedObjects.begin()+objectID); } std::vector VideoTracker::get_all_objects_names() { std::vector objectNames; for (unsigned int i = 0; i < trackedObjects.size(); i++) objectNames.push_back(get_object_name(i)); return objectNames; } void VideoTracker::erase_object_trajectories_to_comply() { for (auto object: trackedObjects) { object->erase_trajectory_to_comply(); } }