NBB.Post.create() too slow, can I avoid it

[akhoury]

NBB.Post.create() too slow, can I avoid it

[barisusakli]

I noticed this slowness in my testing too, the biggest two offenders were :

feed.updateTopic(tid);
feed.updateRecent();

https://github.com/designcreateplay/NodeBB/blob/master/src/posts.js#L73

Commenting them out really sped up post creation.

[akhoury]

oh, thanks, ill test that out

[akhoury]

woah ! i made these changes, along with taking out the part where each post.content's get parsed, and I was able to migrate 350k+ records in 13 minutes on my macbook, it was taking hours, 6 or more before then, i honestly never was able to finish a full run before i got bored.

do you think we can 'rewrite' Post.create, refactor it a little, maybe have an essential Post._create which the importer can use, and Post.create will call Post._create along with the other feed stuff and the plugin.firehook ?

    Posts.create = function(uid, tid, content, callback) {
        if (uid === null) {
            return callback(new Error('invalid-user'), null);
        }

        topics.isLocked(tid, function(err, locked) {
            if(err) {
                return callback(err, null);
            } else if(locked) {
                return callback(new Error('topic-locked'), null);
            }

            db.incrObjectField('global', 'nextPid', function(err, pid) {
                if(err) {
                    return callback(err, null);
                }

                plugins.fireHook('filter:post.save', content, function(err, newContent) {
                    if(err) {
                        return callback(err, null);
                    }

                    var timestamp = Date.now(),
                        postData = {
                            'pid': pid,
                            'uid': uid,
                            'tid': tid,
                            'content': newContent,
                            'timestamp': timestamp,
                            'reputation': 0,
                            'editor': '',
                            'edited': 0,
                            'deleted': 0
                        };

                    db.setObject('post:' + pid, postData);

                    postData.favourited = false;
                    postData.display_moderator_tools = true;
                    postData.relativeTime = new Date(timestamp).toISOString();

                    topics.addPostToTopic(tid, pid);
                    topics.increasePostCount(tid);
                    topics.updateTimestamp(tid, timestamp);

                    db.incrObjectField('global', 'postCount');

                    topics.getTopicFields(tid, ['cid', 'pinned'], function(err, topicData) {

                        var cid = topicData.cid;

                        // [import-temp] uncomment after migration
                        // feed.updateTopic(tid);
                        // feed.updateRecent();

                        db.sortedSetAdd('categories:recent_posts:cid:' + cid, timestamp, pid);

                        if(parseInt(topicData.pinned, 10) === 0) {
                            db.sortedSetAdd('categories:' + cid + ':tid', timestamp, tid);
                        }

                        db.setCount('cid:' + cid + ':active_users', function(err, amount) {
                            if (amount > 15) {
                                db.setRemoveRandom('cid:' + cid + ':active_users');
                            }

                            categories.addActiveUser(cid, uid);
                        });
                    });

                    user.onNewPostMade(uid, tid, pid, timestamp);

                    // [import-temp], delete this statement and uncomment block below after migration
                    callback(null, postData);

//                  plugins.fireHook('filter:post.get', postData, function(err, newPostData) {
//                      if(err) {
//                          return callback(err, null);
//                      }
//
//                      postData = newPostData;
//
//                      postTools.parse(postData.content, function(err, content) {
//                          if(err) {
//                              return callback(err, null);
//                          }
//                          postData.content = content;
//
//                          plugins.fireHook('action:post.save', postData);
//
//                          db.searchIndex('post', content, pid);
//
//                          callback(null, postData);
//                      });
//                  });
                });
            });
        });
    };