Merged in feature_update_users_privacy (pull request #15)

User privacy status update implemented
This commit is contained in:
Alexey Eschenko 2017-11-04 19:51:08 +00:00
commit ad1945b0b6
14 changed files with 413 additions and 131 deletions

View file

@ -0,0 +1,40 @@
<?php
namespace Application\Migrations;
use Doctrine\DBAL\Migrations\AbstractMigration;
use Doctrine\DBAL\Schema\Schema;
/**
* New fields for User entity: 'public' and 'whitelistOnly' (privacy support)
*/
class Version20171104182713 extends AbstractMigration
{
/**
* @param Schema $schema
*/
public function up(Schema $schema)
{
// this up() migration is auto-generated, please modify it to your needs
$this->abortIf($this->connection->getDatabasePlatform()->getName() !== 'postgresql', 'Migration can only be executed safely on \'postgresql\'.');
$this->addSql('ALTER TABLE users.users ADD public BOOLEAN DEFAULT FALSE NOT NULL');
$this->addSql('ALTER TABLE users.users ADD whitelist_only BOOLEAN DEFAULT FALSE NOT NULL');
$this->addSql('CREATE INDEX idx_user_public ON users.users (public)');
$this->addSql('CREATE INDEX idx_user_removed ON users.users (is_removed)');
}
/**
* @param Schema $schema
*/
public function down(Schema $schema)
{
// this down() migration is auto-generated, please modify it to your needs
$this->abortIf($this->connection->getDatabasePlatform()->getName() !== 'postgresql', 'Migration can only be executed safely on \'postgresql\'.');
$this->addSql('DROP INDEX users.idx_user_public');
$this->addSql('DROP INDEX users.idx_user_removed');
$this->addSql('ALTER TABLE users.users DROP public');
$this->addSql('ALTER TABLE users.users DROP whitelist_only');
}
}

View file

@ -38,8 +38,10 @@ class RestoreRemovedUsersCommand extends Command
*/
private $delay;
public function setDependencies(LoggerInterface $logger, EntityManagerInterface $em, UserRepository $userRepo, UserApi $userApi, int $delay): void
public function __construct(LoggerInterface $logger, EntityManagerInterface $em, UserRepository $userRepo, UserApi $userApi, int $delay)
{
parent::__construct();
$this->logger = $logger;
$this->em = $em;
$this->userRepo = $userRepo;
@ -77,6 +79,8 @@ class RestoreRemovedUsersCommand extends Command
'login' => $removedUser->getLogin(),
]);
$removedUser->restore();
$this->em->flush();
}
} catch (UserNotFoundException $e) {
$this->logger->debug('User is really removed. Keep going.', [

View file

@ -4,16 +4,13 @@ namespace Skobkin\Bundle\PointToolsBundle\Command;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Skobkin\Bundle\PointToolsBundle\Entity\Subscription;
use Skobkin\Bundle\PointToolsBundle\Entity\User;
use Skobkin\Bundle\PointToolsBundle\Entity\{Subscription, User};
use Skobkin\Bundle\PointToolsBundle\Exception\Api\UserNotFoundException;
use Skobkin\Bundle\PointToolsBundle\Repository\UserRepository;
use Skobkin\Bundle\PointToolsBundle\Service\SubscriptionsManager;
use Skobkin\Bundle\PointToolsBundle\Service\Api\UserApi;
use Skobkin\Bundle\PointToolsBundle\Service\{SubscriptionsManager, Api\UserApi};
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\{InputInterface, InputOption};
use Symfony\Component\Console\Output\OutputInterface;
/**
@ -51,6 +48,11 @@ class UpdateSubscriptionsCommand extends ContainerAwareCommand
*/
private $apiDelay = 500000;
/**
* @var int
*/
private $appUserId;
/**
* @var SubscriptionsManager
*/
@ -61,19 +63,24 @@ class UpdateSubscriptionsCommand extends ContainerAwareCommand
*/
private $progress;
public function __construct(
EntityManagerInterface $em,
LoggerInterface $logger,
UserRepository $userRepo,
UserApi $api,
SubscriptionsManager $subscriptionManager,
int $apiDelay,
int $appUserId
) {
parent::__construct();
public function setDeps(LoggerInterface $logger, EntityManagerInterface $em, UserRepository $userRepo, UserApi $userApi, SubscriptionsManager $subscriptionsManager): void
{
$this->logger = $logger;
$this->em = $em;
$this->logger = $logger;
$this->userRepo = $userRepo;
$this->api = $userApi;
$this->subscriptionManager = $subscriptionsManager;
}
public function setApiDelay(int $microSecs): void
{
$this->apiDelay = $microSecs;
$this->api = $api;
$this->subscriptionManager = $subscriptionManager;
$this->apiDelay = $apiDelay;
$this->appUserId = $appUserId;
}
protected function configure()
@ -96,35 +103,21 @@ class UpdateSubscriptionsCommand extends ContainerAwareCommand
;
}
/**
* @param InputInterface $input
* @param OutputInterface $output
*
* @return int
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->input = $input;
$this->logger->debug('UpdateSubscriptionsCommand started.');
try {
$appUserId = $this->getContainer()->getParameter('point_id');
} catch (\InvalidArgumentException $e) {
$this->logger->alert('Could not get point_id parameter from config file', ['exception_message' => $e->getMessage()]);
return 1;
}
$this->progress = new ProgressBar($output);
$this->progress->setFormat('debug');
// Beginning transaction for all changes
if ($input->getOption('check-only')) { // Beginning transaction for all changes
$this->em->beginTransaction();
$this->progress->setMessage('Getting service subscribers');
}
try {
$usersForUpdate = $this->getUsersForUpdate($appUserId);
$usersForUpdate = $this->getUsersForUpdate();
} catch (\Exception $e) {
$this->logger->error('Error while getting service subscribers', ['exception' => get_class($e), 'message' => $e->getMessage()]);
@ -138,42 +131,40 @@ class UpdateSubscriptionsCommand extends ContainerAwareCommand
}
$this->logger->info('Processing users subscribers');
$this->progress->setMessage('Processing users subscribers');
$this->progress->start(count($usersForUpdate));
$this->updateUsersSubscribers($usersForUpdate);
foreach ($usersForUpdate as $user) {
usleep($this->apiDelay);
$this->progress->advance();
$this->logger->info('Processing @'.$user->getLogin());
$this->updateUser($user);
}
$this->progress->finish();
// Flushing all changes at once to database
if ($input->getOption('check-only')) { // Flushing all changes at once to the database
$this->em->flush();
$this->em->commit();
}
$this->logger->debug('Finished');
return 0;
}
/**
* @param User[] $users
*/
private function updateUsersSubscribers(array $users): void
private function updateUser(User $user): void
{
// Updating users subscribers
foreach ($users as $user) {
usleep($this->apiDelay);
$this->progress->advance();
$this->logger->info('Processing @'.$user->getLogin());
try {
$userCurrentSubscribers = $this->api->getUserSubscribersById($user->getId());
} catch (UserNotFoundException $e) {
$this->logger->warning('User not found. Marking as removed', ['login' => $user->getLogin(), 'user_id' => $user->getId()]);
$this->logger->warning('User not found. Marking as removed.', ['login' => $user->getLogin(), 'user_id' => $user->getId()]);
$user->markAsRemoved();
continue;
return;
} catch (\Exception $e) {
$this->logger->error(
'Error while getting subscribers. Skipping.',
@ -186,7 +177,7 @@ class UpdateSubscriptionsCommand extends ContainerAwareCommand
]
);
continue;
return;
}
$this->logger->debug('Updating user subscribers');
@ -207,9 +198,8 @@ class UpdateSubscriptionsCommand extends ContainerAwareCommand
);
}
}
}
private function getUsersForUpdate(int $appUserId): array
private function getUsersForUpdate(): array
{
$usersForUpdate = [];
@ -218,7 +208,7 @@ class UpdateSubscriptionsCommand extends ContainerAwareCommand
} else {
/** @var User $serviceUser */
try {
$serviceUser = $this->userRepo->findActiveUserWithSubscribers($appUserId);
$serviceUser = $this->userRepo->findActiveUserWithSubscribers($this->appUserId);
} catch (\Exception $e) {
$this->logger->error('Error while getting active user with subscribers', ['app_user_id' => $appUserId]);
@ -235,7 +225,7 @@ class UpdateSubscriptionsCommand extends ContainerAwareCommand
$this->logger->info('Getting service subscribers');
try {
$usersForUpdate = $this->api->getUserSubscribersById($appUserId);
$usersForUpdate = $this->api->getUserSubscribersById($this->appUserId);
} catch (UserNotFoundException $e) {
$this->logger->critical('Service user deleted or API response is invalid');

View file

@ -0,0 +1,200 @@
<?php
namespace Skobkin\Bundle\PointToolsBundle\Command;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Log\LoggerInterface;
use Skobkin\Bundle\PointToolsBundle\Entity\{Subscription, User};
use Skobkin\Bundle\PointToolsBundle\Exception\Api\{ForbiddenException, UserNotFoundException};
use Skobkin\Bundle\PointToolsBundle\Repository\UserRepository;
use Skobkin\Bundle\PointToolsBundle\Service\Api\UserApi;
use Symfony\Bundle\FrameworkBundle\Command\ContainerAwareCommand;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\{InputInterface, InputOption};
use Symfony\Component\Console\Output\OutputInterface;
class UpdateUsersPrivacyCommand extends ContainerAwareCommand
{
/** @var EntityManagerInterface */
private $em;
/** @var LoggerInterface */
private $logger;
/** @var UserRepository */
private $userRepo;
/** @var InputInterface */
private $input;
/** @var UserApi */
private $api;
/** @var int */
private $apiDelay = 500000;
/** @var int */
private $appUserId;
/** @var ProgressBar */
private $progress;
public function __construct(EntityManagerInterface $em, LoggerInterface $logger, UserRepository $userRepo, UserApi $api, int $apiDelay, int $appUserId)
{
parent::__construct();
$this->em = $em;
$this->logger = $logger;
$this->userRepo = $userRepo;
$this->api = $api;
$this->apiDelay = $apiDelay;
$this->appUserId = $appUserId;
}
/**
* {@inheritdoc}
*/
protected function configure()
{
$this
->setName('point:update:privacy')
->setDescription('Update users privacy')
->addOption(
'all-users',
null,
InputOption::VALUE_NONE,
'If set, command will check all users instead of service subscribers only'
)
;
}
/**
* {@inheritdoc}
*/
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->input = $input;
$this->logger->debug(static::class.' started.');
$this->progress = new ProgressBar($output);
$this->progress->setFormat('debug');
try {
/** @var User[] $usersForUpdate */
$usersForUpdate = $this->getUsersForUpdate();
} catch (\Exception $e) {
$this->logger->error('Error while getting service subscribers', ['exception' => get_class($e), 'message' => $e->getMessage()]);
return 1;
}
$this->logger->info('Processing users privacy.');
$this->progress->start(count($usersForUpdate));
foreach ($usersForUpdate as $idx => $user) {
usleep($this->apiDelay);
$this->progress->advance();
$this->logger->info('Processing @'.$user->getLogin());
$this->updateUser($user);
// Flushing each 10 users
if (0 === $idx % 10) {
$this->em->flush();
}
}
$this->progress->finish();
$this->em->flush();
$this->logger->debug('Finished');
return 0;
}
private function updateUser(User $user): void
{
try {
$remoteUser = $this->api->getUserById($user->getId());
if ($remoteUser !== $user) {
$this->logger->error('Remote user is not equal with local.', ['user_id' => $user->getId(), 'user_login' => $user->getLogin()]);
}
} catch (UserNotFoundException $e) {
$this->logger->info('User not found. Marking as removed.', ['user_id' => $user->getId(), 'user_login' => $user->getLogin()]);
$user->markAsRemoved();
} catch (ForbiddenException $e) {
$this->logger->info('User profile access forbidden', ['user_id' => $user->getId(), 'user_login' => $user->getLogin()]);
$user->updatePrivacy(false, true);
} catch (\Exception $e) {
$this->logger->error(
'Error while updating user privacy',
[
'user_login' => $user->getLogin(),
'user_id' => $user->getId(),
'message' => $e->getMessage(),
'file' => $e->getFile(),
'line' => $e->getLine(),
]
);
}
}
private function getUsersForUpdate(): array
{
if ($this->input->getOption('all-users')) {
return $this->userRepo->findBy(['removed' => false]);
}
/** @var User $serviceUser */
try {
$serviceUser = $this->userRepo->findActiveUserWithSubscribers($this->appUserId);
} catch (\Exception $e) {
$this->logger->error('Error while getting active user with subscribers', ['app_user_id' => $this->appUserId]);
throw $e;
}
if (!$serviceUser) {
$this->logger->critical('Service user not found or marked as removed');
throw new \RuntimeException('Service user not found in the database');
}
$this->logger->info('Getting service subscribers');
try {
return $this->api->getUserSubscribersById($this->appUserId);
} catch (UserNotFoundException $e) {
$this->logger->critical('Service user deleted or API response is invalid');
throw $e;
} catch (\Exception $e) {
$this->logger->warning(
'Error while getting service subscribers. Fallback to local list.',
[
'user_login' => $serviceUser->getLogin(),
'user_id' => $serviceUser->getId(),
'message' => $e->getMessage(),
'file' => $e->getFile(),
'line' => $e->getLine(),
]
);
$localSubscribers = [];
/** @var Subscription $subscription */
foreach ($serviceUser->getSubscribers() as $subscription) {
$localSubscribers[] = $subscription->getSubscriber();
}
return $localSubscribers;
}
}
}

View file

@ -2,9 +2,9 @@
namespace Skobkin\Bundle\PointToolsBundle\Controller\Api;
use Skobkin\Bundle\PointToolsBundle\DTO\Api\PostsPage;
use Skobkin\Bundle\PointToolsBundle\Service\Factory\Blogs\PostFactory;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\HttpFoundation\{Request, Response};
class CrawlerController extends AbstractApiController
{
@ -21,7 +21,7 @@ class CrawlerController extends AbstractApiController
$serializer = $this->get('jms_serializer');
$page = $serializer->deserialize($json, 'Skobkin\Bundle\PointToolsBundle\DTO\Api\PostsPage', 'json');
$page = $serializer->deserialize($json, PostsPage::class, 'json');
/** @var PostFactory $factory */
$factory = $this->get('app.point.post_factory');

View file

@ -6,15 +6,18 @@ use Doctrine\Common\Collections\ArrayCollection;
use Doctrine\ORM\Mapping as ORM;
/**
* @ORM\Table(name="users", schema="users")
* @ORM\Table(name="users", schema="users", indexes={
* @ORM\Index(name="idx_user_public", columns={"public"}),
* @ORM\Index(name="idx_user_removed", columns={"is_removed"})
* })
* @ORM\Entity(repositoryClass="Skobkin\Bundle\PointToolsBundle\Repository\UserRepository")
* @ORM\HasLifecycleCallbacks
*/
class User
{
const AVATAR_SIZE_SMALL = '24';
const AVATAR_SIZE_MEDIUM = '40';
const AVATAR_SIZE_LARGE = '80';
public const AVATAR_SIZE_SMALL = '24';
public const AVATAR_SIZE_MEDIUM = '40';
public const AVATAR_SIZE_LARGE = '80';
/**
* @var int
@ -52,6 +55,20 @@ class User
*/
private $updatedAt;
/**
* @var bool
*
* @ORM\Column(name="public", type="boolean", nullable=false, options={"default": false})
*/
private $public = false;
/**
* @var bool
*
* @ORM\Column(name="whitelist_only", type="boolean", nullable=false, options={"default": false})
*/
private $whitelistOnly = false;
/**
* @var ArrayCollection|Subscription[]
*
@ -140,7 +157,7 @@ class User
/**
* @return Subscription[]|ArrayCollection
*/
public function getSubscribers(): iterable
public function getSubscribers(): ArrayCollection
{
return $this->subscribers;
}
@ -148,7 +165,7 @@ class User
/**
* @return Subscription[]|ArrayCollection
*/
public function getSubscriptions(): iterable
public function getSubscriptions(): ArrayCollection
{
return $this->subscriptions;
}
@ -163,7 +180,7 @@ class User
/**
* @return SubscriptionEvent[]|ArrayCollection
*/
public function getNewSubscriberEvents(): iterable
public function getNewSubscriberEvents(): ArrayCollection
{
return $this->newSubscriberEvents;
}
@ -178,6 +195,22 @@ class User
return $this->updatedAt;
}
public function updatePrivacy(?bool $public, ?bool $whitelistOnly): void
{
$this->public = $public;
$this->whitelistOnly = $whitelistOnly;
}
public function isPublic(): ?bool
{
return $this->public;
}
public function isWhitelistOnly(): ?bool
{
return $this->whitelistOnly;
}
public function isRemoved(): bool
{
return $this->removed;

View file

@ -2,7 +2,7 @@
namespace Skobkin\Bundle\PointToolsBundle\Exception\Api;
class UserNotFoundException extends ApiException
class UserNotFoundException extends NotFoundException
{
/**
* @var int
@ -22,6 +22,7 @@ class UserNotFoundException extends ApiException
public function __construct($message = 'User not found', $code = 0, \Exception $previous = null, $userId = null, $login = null)
{
parent::__construct($message, $code, $previous);
$this->userId = $userId;
$this->login = $login;
}

View file

@ -17,7 +17,7 @@ class UserRepository extends EntityRepository
{
$qb = $this->createQueryBuilder('u');
// May be optimize hydration procedure
// @todo May be optimize hydration procedure
return $qb
->select(['u', 's', 'us'])
->innerJoin('u.subscribers', 's')

View file

@ -53,18 +53,40 @@ services:
# Subsribers update
app.point.update_subscribers_command:
class: Skobkin\Bundle\PointToolsBundle\Command\UpdateSubscriptionsCommand
#autowire: []
calls:
- [setDeps, ['@logger', '@doctrine.orm.entity_manager', '@app.point.user_repository', '@app.point.api_user', '@app.point.subscriptions_manager']]
- [setApiDelay, ['%point_api_delay%']]
arguments:
- '@doctrine.orm.entity_manager'
- '@logger'
- '@app.point.user_repository'
- '@app.point.api_user'
- '@app.point.subscriptions_manager'
- '%point_api_delay%'
- '%point_id%'
tags:
- { name: console.command }
- { name: monolog.logger, channel: subscribers_update }
# Privacy update
app.point.update_privacy_command:
class: Skobkin\Bundle\PointToolsBundle\Command\UpdateUsersPrivacyCommand
#autowire: []
arguments:
- '@doctrine.orm.entity_manager'
- '@logger'
- '@app.point.user_repository'
- '@app.point.api_user'
- '%point_api_delay%'
- '%point_id%'
tags:
- { name: console.command }
- { name: monolog.logger, channel: privacy_update }
# Restore users removed by error
app.point.restore_users:
class: Skobkin\Bundle\PointToolsBundle\Command\RestoreRemovedUsersCommand
calls:
- [setDependencies, ['@logger', '@doctrine.orm.entity_manager', '@app.point.user_repository', '@app.point.api_user', '%point_api_delay%']]
arguments:
- '@logger'
- '@doctrine.orm.entity_manager'
- '@app.point.user_repository'
- '@app.point.api_user'
- '%point_api_delay%'
tags:
- { name: console.command }
# Webhook management

View file

@ -4,16 +4,10 @@ namespace Skobkin\Bundle\PointToolsBundle\Service\Api;
use GuzzleHttp\ClientInterface;
use GuzzleHttp\Exception\TransferException;
use JMS\Serializer\DeserializationContext;
use JMS\Serializer\Serializer;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\StreamInterface;
use JMS\Serializer\{DeserializationContext, Serializer};
use Psr\Http\Message\{ResponseInterface, StreamInterface};
use Psr\Log\LoggerInterface;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\ForbiddenException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\NetworkException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\NotFoundException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\ServerProblemException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\UnauthorizedException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\{ForbiddenException, NetworkException, NotFoundException, ServerProblemException, UnauthorizedException};
use Symfony\Component\HttpFoundation\Response as SymfonyResponse;
class AbstractApi
@ -184,7 +178,9 @@ class AbstractApi
// @todo remove after fix
// Temporary fix until @arts fixes this bug
if ('{"error": "UserNotFound"}' === (string) $response->getBody()) {
throw new NotFoundException($reason, $code);
throw new NotFoundException('Not found', SymfonyResponse::HTTP_NOT_FOUND);
} elseif ('{"message": "Forbidden", "code": 403, "error": "Forbidden"}' === (string) $response->getBody()) {
throw new ForbiddenException('Forbidden', SymfonyResponse::HTTP_FORBIDDEN);
}
switch ($code) {

View file

@ -3,16 +3,11 @@
namespace Skobkin\Bundle\PointToolsBundle\Service\Api;
use GuzzleHttp\ClientInterface;
use JMS\Serializer\DeserializationContext;
use JMS\Serializer\Serializer;
use JMS\Serializer\{DeserializationContext, Serializer};
use Psr\Log\LoggerInterface;
use Skobkin\Bundle\PointToolsBundle\DTO\Api\Auth;
use Skobkin\Bundle\PointToolsBundle\DTO\Api\User as UserDTO;
use Skobkin\Bundle\PointToolsBundle\DTO\Api\{Auth, User as UserDTO};
use Skobkin\Bundle\PointToolsBundle\Entity\User;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\ForbiddenException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\InvalidResponseException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\NotFoundException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\UserNotFoundException;
use Skobkin\Bundle\PointToolsBundle\Exception\Api\{ForbiddenException, InvalidResponseException, NotFoundException, UserNotFoundException};
use Skobkin\Bundle\PointToolsBundle\Service\Factory\UserFactory;
/**
@ -177,6 +172,7 @@ class UserApi extends AbstractApi
} catch (NotFoundException $e) {
throw new UserNotFoundException('User not found', 0, $e, $id);
}
// Not catching ForbiddenException right now
return $this->userFactory->findOrCreateFromDTO($userData);
}

View file

@ -48,6 +48,10 @@ class UserFactory extends AbstractFactory
$user->updateLoginAndName($userData->getLogin(), $userData->getName());
if (null !== $userData->getDenyAnonymous() && null !== $userData->getPrivate()) {
$user->updatePrivacy(!$userData->getDenyAnonymous(), $userData->getPrivate());
}
return $user;
}

View file

@ -2,9 +2,7 @@
namespace Skobkin\Bundle\PointToolsBundle\Service\Telegram;
use unreal4u\TelegramAPI\Telegram\Types\Inline\Query;
use unreal4u\TelegramAPI\Telegram\Types\Message;
use unreal4u\TelegramAPI\Telegram\Types\Update;
use unreal4u\TelegramAPI\Telegram\Types\{Inline\Query, Message, Update};
/**
* Dispatches incoming messages processing to corresponding services

View file

@ -2,9 +2,7 @@
namespace Skobkin\Bundle\PointToolsBundle\Service\Telegram;
use Skobkin\Bundle\PointToolsBundle\Entity\Telegram\Account;
use Skobkin\Bundle\PointToolsBundle\Entity\User;
use Skobkin\Bundle\PointToolsBundle\Entity\UserRenameEvent;
use Skobkin\Bundle\PointToolsBundle\Entity\{Telegram\Account, User, UserRenameEvent};
use Skobkin\Bundle\PointToolsBundle\Repository\Telegram\AccountRepository;
/**