From a1b982d8914788a24b6a5f032145d8874b9ae8ce Mon Sep 17 00:00:00 2001 From: Alexey Skobkin Date: Thu, 24 Mar 2016 01:49:22 +0300 Subject: [PATCH] Test implementation of crawler API. NOT SECURED BY TOKEN YET. --- .../Controller/Api/CrawlerController.php | 8 +- .../PointToolsBundle/Entity/Blogs/Post.php | 5 +- .../Resources/config/services.yml | 4 +- .../Service/Factory/Blogs/PostFactory.php | 192 ++++++++++++------ .../Service/Factory/Blogs/TagFactory.php | 80 ++++---- 5 files changed, 181 insertions(+), 108 deletions(-) diff --git a/src/Skobkin/Bundle/PointToolsBundle/Controller/Api/CrawlerController.php b/src/Skobkin/Bundle/PointToolsBundle/Controller/Api/CrawlerController.php index fba2400..b4eca16 100644 --- a/src/Skobkin/Bundle/PointToolsBundle/Controller/Api/CrawlerController.php +++ b/src/Skobkin/Bundle/PointToolsBundle/Controller/Api/CrawlerController.php @@ -2,6 +2,7 @@ namespace Skobkin\Bundle\PointToolsBundle\Controller\Api; +use Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs\PostFactory; use Symfony\Component\HttpFoundation\Request; class CrawlerController extends AbstractApiController @@ -15,8 +16,13 @@ class CrawlerController extends AbstractApiController $page = $serializer->deserialize($json, 'Skobkin\Bundle\PointToolsBundle\DTO\Api\Crawler\PostsPage', 'json'); + /** @var PostFactory $factory */ + $factory = $this->get('skobkin__point_tools.service_factory.post_factory'); + + $continue = $factory->createFromPageDTO($page); + return $this->createSuccessResponse([ - 'continue' => false, + 'continue' => $continue, ]); } } diff --git a/src/Skobkin/Bundle/PointToolsBundle/Entity/Blogs/Post.php b/src/Skobkin/Bundle/PointToolsBundle/Entity/Blogs/Post.php index 737a6ae..b175fd8 100644 --- a/src/Skobkin/Bundle/PointToolsBundle/Entity/Blogs/Post.php +++ b/src/Skobkin/Bundle/PointToolsBundle/Entity/Blogs/Post.php @@ -46,7 +46,7 @@ class Post /** * @var \DateTime * - * @ORM\Column(name="updated_at", type="datetime") + * @ORM\Column(name="updated_at", type="datetime", nullable=true) */ private $updatedAt; @@ -55,7 +55,7 @@ class Post * * @ORM\Column(name="type", type="string", length=6) */ - private $type; + private $type = self::TYPE_POST; /** * @var bool @@ -227,6 +227,7 @@ class Post */ public function addPostTag(PostTag $tag) { + $tag->setPost($this); $this->postTags[] = $tag; return $this; diff --git a/src/Skobkin/Bundle/PointToolsBundle/Resources/config/services.yml b/src/Skobkin/Bundle/PointToolsBundle/Resources/config/services.yml index 931fb16..8e15948 100644 --- a/src/Skobkin/Bundle/PointToolsBundle/Resources/config/services.yml +++ b/src/Skobkin/Bundle/PointToolsBundle/Resources/config/services.yml @@ -35,8 +35,8 @@ services: skobkin__point_tools.service_factory.tag_factory: class: Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs\TagFactory - arguments: [ @doctrine.orm.entity_manager ] + arguments: [ @logger, @doctrine.orm.entity_manager ] skobkin__point_tools.service_factory.post_factory: class: Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs\PostFactory - arguments: [ @doctrine.orm.entity_manager, @skobkin__point_tools.service_factory.user_factory, @skobkin__point_tools.service_factory.comment_factory, @skobkin__point_tools.service_factory.tag_factory ] + arguments: [ @logger, @doctrine.orm.entity_manager, @skobkin__point_tools.service_factory.user_factory, @skobkin__point_tools.service_factory.comment_factory, @skobkin__point_tools.service_factory.tag_factory ] diff --git a/src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/PostFactory.php b/src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/PostFactory.php index d0e5e45..27c8709 100644 --- a/src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/PostFactory.php +++ b/src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/PostFactory.php @@ -5,14 +5,25 @@ namespace Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs; use Doctrine\ORM\EntityManager; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\EntityRepository; +use Psr\Log\LoggerInterface; +use Skobkin\Bundle\PointToolsBundle\DTO\Api\Crawler\MetaPost; +use Skobkin\Bundle\PointToolsBundle\DTO\Api\Crawler\PostsPage; use Skobkin\Bundle\PointToolsBundle\Entity\Blogs\Post; +use Skobkin\Bundle\PointToolsBundle\Entity\Blogs\PostTag; +use Skobkin\Bundle\PointToolsBundle\Entity\Blogs\Tag; use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\ApiException; +use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\Factory\Blogs\InvalidPostDataException; use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\InvalidResponseException; use Skobkin\Bundle\PointToolsBundle\Service\Factory\UserFactory; class PostFactory { + /** + * @var LoggerInterface + */ + private $log; + /** * @var EntityManager */ @@ -41,8 +52,9 @@ class PostFactory /** * @param EntityManager $em */ - public function __construct(EntityManagerInterface $em, UserFactory $userFactory, CommentFactory $commentFactory, TagFactory $tagFactory) + public function __construct(LoggerInterface $log, EntityManagerInterface $em, UserFactory $userFactory, CommentFactory $commentFactory, TagFactory $tagFactory) { + $this->log = $log; $this->userFactory = $userFactory; $this->commentFactory = $commentFactory; $this->tagFactory = $tagFactory; @@ -51,91 +63,145 @@ class PostFactory } /** - * @param array $data + * Creates posts and return status of new insertions * - * @return Post + * @param PostsPage $data + * + * @return bool * @throws ApiException * @throws InvalidResponseException */ - public function createFromArray(array $data) + public function createFromPageDTO(PostsPage $page) { - $this->validateData($data); + $posts = []; - if (null === ($post = $this->postRepository->find($data['post']['id']))) { - $createdAt = new \DateTime($data['post']['created']); - $author = $this->userFactory->createFromArray($data['post']['author']); + $hasNew = false; - $post = new Post($data['post']['id'], $data['post']['type'], $data['post']['text'], $createdAt, $author); + foreach ($page->getPosts() as $postData) { + try { + if (null === $this->postRepository->find($postData->getPost()->getId())) { + $hasNew = true; + } + + $post = $this->createFromDTO($postData); + $posts[] = $post; + } catch (\Exception $e) { + $this->log->error('Error while processing post DTO', ['post_id' => $postData->getPost()->getId()]); + } + } + + foreach ($posts as $post) { + if ($this->em->getUnitOfWork()->isScheduledForInsert($post)) { + $hasNew = true; + } + } + + $this->em->flush(); + + return $hasNew; + } + + /** + * @todo Implement full post with comments processing + * + * @param MetaPost $postData + * + * @return Post + * @throws ApiException + * @throws InvalidPostDataException + */ + private function createFromDTO(MetaPost $postData) + { + if (!$this->validateMetaPost($postData)) { + throw new InvalidPostDataException('Invalid post data', $postData); + } + + if (!$postData->getPost()->getAuthor()->getId()) { + $this->log->error('Post author does not contain id', ['post_id' => $postData->getPost()->getId()]); + throw new InvalidPostDataException('Post author does not contain id', $postData); + } + + $user = $this->userFactory->createFromDTO($postData->getPost()->getAuthor()); + + if (null === ($post = $this->postRepository->find($postData->getPost()->getId()))) { + // Creating new post + $post = new Post($postData->getPost()->getId()); $this->em->persist($post); } - $post->setText($data['post']['text']); + // Updating data + $post + ->setAuthor($user) + ->setCreatedAt((new \DateTime($postData->getPost()->getCreated())) ?: null) + // @fixme Need bugfix for point API (type is not showing now) + //->setType($postData->getPost()->getType()) + ->setText($postData->getPost()->getText()) + ->setPrivate($postData->getPost()->getPrivate()) + ; - // Tags - $tags = $this->tagFactory->createFromListArray($data['post']['tags']); + $this->updatePostTags($post, $postData->getPost()->getTags()); - // Removing deleted tags - foreach ($post->getTags() as $tag) { - if (false === in_array($tag, $tags, true)) { - $post->removeTag($tag); - } - } - - // Adding new tags - foreach ($tags as $tag) { - if (!$post->getTags()->contains($tag)) { - $post->addTag($tag); - } - } - - // Flushing post before linking comments - try { - $this->em->flush($post); - } catch (\Exception $e) { - throw new ApiException(sprintf('Error while flushing changes for #%s: %s', $data['post']['id'], $e->getMessage()), 0, $e); - } - - // Comments - $comments = $this->commentFactory->createFromListArray($data['comments']); - - // Marking removed comments - foreach ($post->getComments() as $comment) { - if (false === in_array($comment, $comments, true)) { - $comment->setDeleted(true); - } - } - - // Adding comments - foreach ($comments as $comment) { - $post->addComment($comment); - } + $this->em->flush($post); return $post; } /** - * @param array $data - * - * @throws InvalidResponseException + * @param Post $post + * @param Tag[] $tags */ - private function validateData(array $data) + private function updatePostTags(Post $post, array $tagsStrings) { - if (!array_key_exists('post', $data)) { - throw new InvalidResponseException('Post data not found in API response'); + $tags = $this->tagFactory->createFromStringsArray($tagsStrings); + + // Hashing tags strings + $tagStringsHash = []; + foreach ($tagsStrings as $tagsString) { + $tagStringsHash[mb_strtolower($tagsString)] = $tagsString; } - if (!array_key_exists('comments', $data)) { - throw new InvalidResponseException('Comments data not found in API response'); + // Hashing current post tags + $newTagsHash = []; + foreach ($tags as $tag) { + $newTagsHash[mb_strtolower($tag->getText())] = $tag; } - if (!( - array_key_exists('id', $data['post']) && - array_key_exists('type', $data['post']) && - array_key_exists('text', $data['post']) && - array_key_exists('tags', $data['post']) && - array_key_exists('author', $data['post']) - )) { - throw new InvalidResponseException('Post content not found in API response'); + // Hashing old post tags (from DB) + $oldTagsHash = []; + foreach ($post->getPostTags() as $postTag) { + $oldTagsHash[mb_strtolower($postTag->getOriginalTagText())] = $postTag; + } + + // Adding missing tags + foreach ($tags as $tag) { + if (!array_key_exists(mb_strtolower($tag->getText()), $oldTagsHash)) { + $tmpPostTag = (new PostTag($tag)) + ->setText($tagStringsHash[mb_strtolower($tag->getText())]) + ; + $post->addPostTag($tmpPostTag); + } + } + + // Removing deleted tags + foreach ($post->getPostTags() as $postTag) { + if (!array_key_exists(mb_strtolower($postTag->getOriginalTagText()), $newTagsHash)) { + $post->removePostTag($postTag); + } } } + + private function validateMetaPost(MetaPost $post) + { + if (!$post->getPost()->getId()) { + $this->log->error('Post DTO contains no id'); + return false; + } + + if (null === $post->getPost()->getAuthor()->getId()) { + $this->log->error('Post DTO contains no valid User DTO.', ['post_id' => $post->getPost()->getId()]); + return false; + } + + return true; + } } \ No newline at end of file diff --git a/src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/TagFactory.php b/src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/TagFactory.php index 2d74cbc..12f18bd 100644 --- a/src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/TagFactory.php +++ b/src/Skobkin/Bundle/PointToolsBundle/Service/Factory/Blogs/TagFactory.php @@ -5,8 +5,8 @@ namespace Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs; use Doctrine\ORM\EntityManager; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\EntityRepository; +use Psr\Log\LoggerInterface; use Skobkin\Bundle\PointToolsBundle\Entity\Blogs\Tag; -use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\ApiException; use Skobkin\Bundle\PointToolsBundle\Service\Exceptions\InvalidResponseException; @@ -17,6 +17,11 @@ class TagFactory */ private $em; + /** + * @var LoggerInterface + */ + private $log; + /** * @var EntityRepository */ @@ -25,62 +30,56 @@ class TagFactory /** * @param EntityManager $em */ - public function __construct(EntityManagerInterface $em) + public function __construct(LoggerInterface $log, EntityManagerInterface $em) { + $this->log = $log; $this->em = $em; $this->tagRepository = $em->getRepository('SkobkinPointToolsBundle:Blogs\Tag'); } /** - * @param $data - * - * @return Tag - * @throws ApiException - * @throws InvalidResponseException - * @throws \Doctrine\ORM\NonUniqueResultException - */ - public function createFromArray($data) - { - $this->validateData($data); - - $qb = $this->tagRepository->createQueryBuilder('t'); - $qb - ->select() - ->where($qb->expr()->eq('lower(t.text)', 'lower(:text)')) - ->setParameter('text', $data) - ; - - if (null === ($tag = $qb->getQuery()->getOneOrNullResult())) { - $tag = new Tag($data); - $this->em->persist($tag); - } - - try { - $this->em->flush($tag); - } catch (\Exception $e) { - throw new ApiException(sprintf('Error while flushing changes for [%d] %s: %s', $tag->getId(), $tag->getText(), $e->getMessage()), 0, $e); - } - - return $tag; - } - - /** - * @param array $data + * @param string[] $tagStrings * * @return Tag[] - * @throws ApiException */ - public function createFromListArray(array $data) + public function createFromStringsArray(array $tagStrings) { $tags = []; - foreach ($data as $text) { - $tags[] = $this->createFromArray($text); + foreach ($tagStrings as $string) { + try { + $tag = $this->createFromString($string); + $tags[] = $tag; + } catch (\Exception $e) { + $this->log->error('Error while creating tag from DTO', ['tag' => $string, 'message' => $e->getMessage()]); + continue; + } } return $tags; } + /** + * @param $text + * + * @return Tag + * @throws InvalidResponseException + */ + public function createFromString($text) + { + $this->validateData($text); + + if (null === ($tag = $this->tagRepository->findOneByLowerText($text))) { + // Creating new tag + $tag = new Tag($text); + $this->em->persist($tag); + } + + $this->em->flush($tag); + + return $tag; + } + /** * @param $data * @@ -89,6 +88,7 @@ class TagFactory private function validateData($data) { if (!is_string($data)) { + // @todo Change exception throw new InvalidResponseException('Tag data must be a string'); } }