Test implementation of crawler API. NOT SECURED BY TOKEN YET.

src/Skobkin/Bundle/PointToolsBundle/Controller/Api/CrawlerController.php

@@ -2,6 +2,7 @@
 
 namespace Skobkin\Bundle\PointToolsBundle\Controller\Api;
 
+use Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs\PostFactory;
 use Symfony\Component\HttpFoundation\Request;
 
 class CrawlerController extends AbstractApiController
@@ -15,8 +16,13 @@ class CrawlerController extends AbstractApiController
 
         $page = $serializer->deserialize($json, 'Skobkin\Bundle\PointToolsBundle\DTO\Api\Crawler\PostsPage', 'json');
 
+        /** @var PostFactory $factory */
+        $factory = $this->get('skobkin__point_tools.service_factory.post_factory');
+        
+        $continue = $factory->createFromPageDTO($page);
+
         return $this->createSuccessResponse([
-            'continue' => false,
+            'continue' => $continue,
         ]);
     }
 }

src/Skobkin/Bundle/PointToolsBundle/Entity/Blogs/Post.php

@@ -46,7 +46,7 @@ class Post
     /**
      * @var \DateTime
      *
-     * @ORM\Column(name="updated_at", type="datetime")
+     * @ORM\Column(name="updated_at", type="datetime", nullable=true)
      */
     private $updatedAt;
 
@@ -55,7 +55,7 @@ class Post
      *
      * @ORM\Column(name="type", type="string", length=6)
      */
-    private $type;
+    private $type = self::TYPE_POST;
 
     /**
      * @var bool
@@ -227,6 +227,7 @@ class Post
      */
     public function addPostTag(PostTag $tag)
     {
+        $tag->setPost($this);
         $this->postTags[] = $tag;
 
         return $this;

src/Skobkin/Bundle/PointToolsBundle/Resources/config/services.yml

@@ -35,8 +35,8 @@ services:
 
     skobkin__point_tools.service_factory.tag_factory:
         class: Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs\TagFactory
-        arguments: [ @doctrine.orm.entity_manager ]
+        arguments: [ @logger, @doctrine.orm.entity_manager ]
 
     skobkin__point_tools.service_factory.post_factory:
         class: Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs\PostFactory
-        arguments: [ @doctrine.orm.entity_manager, @skobkin__point_tools.service_factory.user_factory, @skobkin__point_tools.service_factory.comment_factory, @skobkin__point_tools.service_factory.tag_factory ]
+        arguments: [ @logger, @doctrine.orm.entity_manager, @skobkin__point_tools.service_factory.user_factory, @skobkin__point_tools.service_factory.comment_factory, @skobkin__point_tools.service_factory.tag_factory ]

src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/PostFactory.php

@@ -5,14 +5,25 @@ namespace Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs;
 use Doctrine\ORM\EntityManager;
 use Doctrine\ORM\EntityManagerInterface;
 use Doctrine\ORM\EntityRepository;
+use Psr\Log\LoggerInterface;
+use Skobkin\Bundle\PointToolsBundle\DTO\Api\Crawler\MetaPost;
+use Skobkin\Bundle\PointToolsBundle\DTO\Api\Crawler\PostsPage;
 use Skobkin\Bundle\PointToolsBundle\Entity\Blogs\Post;
+use Skobkin\Bundle\PointToolsBundle\Entity\Blogs\PostTag;
+use Skobkin\Bundle\PointToolsBundle\Entity\Blogs\Tag;
 use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\ApiException;
+use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\Factory\Blogs\InvalidPostDataException;
 use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\InvalidResponseException;
 use Skobkin\Bundle\PointToolsBundle\Service\Factory\UserFactory;
 
 
 class PostFactory
 {
+    /**
+     * @var LoggerInterface
+     */
+    private $log;
+
     /**
      * @var EntityManager
      */
@@ -41,8 +52,9 @@ class PostFactory
     /**
      * @param EntityManager $em
      */
-    public function __construct(EntityManagerInterface $em, UserFactory $userFactory, CommentFactory $commentFactory, TagFactory $tagFactory)
+    public function __construct(LoggerInterface $log, EntityManagerInterface $em, UserFactory $userFactory, CommentFactory $commentFactory, TagFactory $tagFactory)
     {
+        $this->log = $log;
         $this->userFactory = $userFactory;
         $this->commentFactory = $commentFactory;
         $this->tagFactory = $tagFactory;
@@ -51,91 +63,145 @@ class PostFactory
     }
 
     /**
-     * @param array $data
+     * Creates posts and return status of new insertions
      *
-     * @return Post
+     * @param PostsPage $data
+     *
+     * @return bool
      * @throws ApiException
      * @throws InvalidResponseException
      */
-    public function createFromArray(array $data)
+    public function createFromPageDTO(PostsPage $page)
     {
-        $this->validateData($data);
+        $posts = [];
 
-        if (null === ($post = $this->postRepository->find($data['post']['id']))) {
+        $hasNew = false;
-            $createdAt = new \DateTime($data['post']['created']);
-            $author = $this->userFactory->createFromArray($data['post']['author']);
 
-            $post = new Post($data['post']['id'], $data['post']['type'], $data['post']['text'], $createdAt, $author);
+        foreach ($page->getPosts() as $postData) {
+            try {
+                if (null === $this->postRepository->find($postData->getPost()->getId())) {
+                    $hasNew = true;
+                }
+
+                $post = $this->createFromDTO($postData);
+                $posts[] = $post;
+            } catch (\Exception $e) {
+                $this->log->error('Error while processing post DTO', ['post_id' => $postData->getPost()->getId()]);
+            }
+        }
+
+        foreach ($posts as $post) {
+            if ($this->em->getUnitOfWork()->isScheduledForInsert($post)) {
+                $hasNew = true;
+            }
+        }
+
+        $this->em->flush();
+
+        return $hasNew;
+    }
+
+    /**
+     * @todo Implement full post with comments processing
+     *
+     * @param MetaPost $postData
+     *
+     * @return Post
+     * @throws ApiException
+     * @throws InvalidPostDataException
+     */
+    private function createFromDTO(MetaPost $postData)
+    {
+        if (!$this->validateMetaPost($postData)) {
+            throw new InvalidPostDataException('Invalid post data', $postData);
+        }
+
+        if (!$postData->getPost()->getAuthor()->getId()) {
+            $this->log->error('Post author does not contain id', ['post_id' => $postData->getPost()->getId()]);
+            throw new InvalidPostDataException('Post author does not contain id', $postData);
+        }
+
+        $user = $this->userFactory->createFromDTO($postData->getPost()->getAuthor());
+
+        if (null === ($post = $this->postRepository->find($postData->getPost()->getId()))) {
+            // Creating new post
+            $post = new Post($postData->getPost()->getId());
             $this->em->persist($post);
         }
 
-        $post->setText($data['post']['text']);
+        // Updating data
+        $post
+            ->setAuthor($user)
+            ->setCreatedAt((new \DateTime($postData->getPost()->getCreated())) ?: null)
+            // @fixme Need bugfix for point API (type is not showing now)
+            //->setType($postData->getPost()->getType())
+            ->setText($postData->getPost()->getText())
+            ->setPrivate($postData->getPost()->getPrivate())
+        ;
 
-        // Tags
+        $this->updatePostTags($post, $postData->getPost()->getTags());
-        $tags = $this->tagFactory->createFromListArray($data['post']['tags']);
 
-        // Removing deleted tags
+        $this->em->flush($post);
-        foreach ($post->getTags() as $tag) {
-            if (false === in_array($tag, $tags, true)) {
-                $post->removeTag($tag);
-            }
-        }
-
-        // Adding new tags
-        foreach ($tags as $tag) {
-            if (!$post->getTags()->contains($tag)) {
-                $post->addTag($tag);
-            }
-        }
-
-        // Flushing post before linking comments
-        try {
-            $this->em->flush($post);
-        } catch (\Exception $e) {
-            throw new ApiException(sprintf('Error while flushing changes for #%s: %s', $data['post']['id'], $e->getMessage()), 0, $e);
-        }
-
-        // Comments
-        $comments = $this->commentFactory->createFromListArray($data['comments']);
-
-        // Marking removed comments
-        foreach ($post->getComments() as $comment) {
-            if (false === in_array($comment, $comments, true)) {
-                $comment->setDeleted(true);
-            }
-        }
-
-        // Adding comments
-        foreach ($comments as $comment) {
-            $post->addComment($comment);
-        }
 
         return $post;
     }
 
     /**
-     * @param array $data
+     * @param Post $post
-     *
+     * @param Tag[] $tags
-     * @throws InvalidResponseException
      */
-    private function validateData(array $data)
+    private function updatePostTags(Post $post, array $tagsStrings)
     {
-        if (!array_key_exists('post', $data)) {
+        $tags = $this->tagFactory->createFromStringsArray($tagsStrings);
-            throw new InvalidResponseException('Post data not found in API response');
+
+        // Hashing tags strings
+        $tagStringsHash = [];
+        foreach ($tagsStrings as $tagsString) {
+            $tagStringsHash[mb_strtolower($tagsString)] = $tagsString;
         }
 
-        if (!array_key_exists('comments', $data)) {
+        // Hashing current post tags
-            throw new InvalidResponseException('Comments data not found in API response');
+        $newTagsHash = [];
+        foreach ($tags as $tag) {
+            $newTagsHash[mb_strtolower($tag->getText())] = $tag;
         }
 
-        if (!(
+        // Hashing old post tags (from DB)
-            array_key_exists('id', $data['post']) &&
+        $oldTagsHash = [];
-            array_key_exists('type', $data['post']) &&
+        foreach ($post->getPostTags() as $postTag) {
-            array_key_exists('text', $data['post']) &&
+            $oldTagsHash[mb_strtolower($postTag->getOriginalTagText())] = $postTag;
-            array_key_exists('tags', $data['post']) &&
+        }
-            array_key_exists('author', $data['post'])
+
-            )) {
+        // Adding missing tags
-            throw new InvalidResponseException('Post content not found in API response');
+        foreach ($tags as $tag) {
+            if (!array_key_exists(mb_strtolower($tag->getText()), $oldTagsHash)) {
+                $tmpPostTag = (new PostTag($tag))
+                    ->setText($tagStringsHash[mb_strtolower($tag->getText())])
+                ;
+                $post->addPostTag($tmpPostTag);
+            }
+        }
+
+        // Removing deleted tags
+        foreach ($post->getPostTags() as $postTag) {
+            if (!array_key_exists(mb_strtolower($postTag->getOriginalTagText()), $newTagsHash)) {
+                $post->removePostTag($postTag);
+            }
         }
     }
+
+    private function validateMetaPost(MetaPost $post)
+    {
+        if (!$post->getPost()->getId()) {
+            $this->log->error('Post DTO contains no id');
+            return false;
+        }
+
+        if (null === $post->getPost()->getAuthor()->getId()) {
+            $this->log->error('Post DTO contains no valid User DTO.', ['post_id' => $post->getPost()->getId()]);
+            return false;
+        }
+
+        return true;
+    }
 }

src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/TagFactory.php

@@ -5,8 +5,8 @@ namespace Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs;
 use Doctrine\ORM\EntityManager;
 use Doctrine\ORM\EntityManagerInterface;
 use Doctrine\ORM\EntityRepository;
+use Psr\Log\LoggerInterface;
 use Skobkin\Bundle\PointToolsBundle\Entity\Blogs\Tag;
-use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\ApiException;
 use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\InvalidResponseException;
 
 
@@ -17,6 +17,11 @@ class TagFactory
      */
     private $em;
 
+    /**
+     * @var LoggerInterface
+     */
+    private $log;
+
     /**
      * @var EntityRepository
      */
@@ -25,62 +30,56 @@ class TagFactory
     /**
      * @param EntityManager $em
      */
-    public function __construct(EntityManagerInterface $em)
+    public function __construct(LoggerInterface $log, EntityManagerInterface $em)
     {
+        $this->log = $log;
         $this->em = $em;
         $this->tagRepository = $em->getRepository('SkobkinPointToolsBundle:Blogs\Tag');
     }
 
     /**
-     * @param $data
+     * @param string[] $tagStrings
-     *
-     * @return Tag
-     * @throws ApiException
-     * @throws InvalidResponseException
-     * @throws \Doctrine\ORM\NonUniqueResultException
-     */
-    public function createFromArray($data)
-    {
-        $this->validateData($data);
-
-        $qb = $this->tagRepository->createQueryBuilder('t');
-        $qb
-            ->select()
-            ->where($qb->expr()->eq('lower(t.text)', 'lower(:text)'))
-            ->setParameter('text', $data)
-        ;
-
-        if (null === ($tag = $qb->getQuery()->getOneOrNullResult())) {
-            $tag = new Tag($data);
-            $this->em->persist($tag);
-        }
-
-        try {
-            $this->em->flush($tag);
-        } catch (\Exception $e) {
-            throw new ApiException(sprintf('Error while flushing changes for [%d] %s: %s', $tag->getId(), $tag->getText(), $e->getMessage()), 0, $e);
-        }
-
-        return $tag;
-    }
-
-    /**
-     * @param array $data
      *
      * @return Tag[]
-     * @throws ApiException
      */
-    public function createFromListArray(array $data)
+    public function createFromStringsArray(array $tagStrings)
     {
         $tags = [];
 
-        foreach ($data as $text) {
+        foreach ($tagStrings as $string) {
-            $tags[] = $this->createFromArray($text);
+            try {
+                $tag = $this->createFromString($string);
+                $tags[] = $tag;
+            } catch (\Exception $e) {
+                $this->log->error('Error while creating tag from DTO', ['tag' => $string, 'message' => $e->getMessage()]);
+                continue;
+            }
         }
 
         return $tags;
     }
 
+    /**
+     * @param $text
+     *
+     * @return Tag
+     * @throws InvalidResponseException
+     */
+    public function createFromString($text)
+    {
+        $this->validateData($text);
+
+        if (null === ($tag = $this->tagRepository->findOneByLowerText($text))) {
+            // Creating new tag
+            $tag = new Tag($text);
+            $this->em->persist($tag);
+        }
+
+        $this->em->flush($tag);
+
+        return $tag;
+    }
+
     /**
      * @param $data
      *
@@ -89,6 +88,7 @@ class TagFactory
     private function validateData($data)
     {
         if (!is_string($data)) {
+            // @todo Change exception
             throw new InvalidResponseException('Tag data must be a string');
         }
     }