diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 01a95103..a682acc1 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -678,7 +678,6 @@ Style/RescueStandardError: - 'app/lib/json_web_token.rb' - 'app/lib/send_grid_client.rb' - 'config/initializers/core_extensions/devise/strategies/json_web_token.rb' - - 'lib/tasks/git_hub.rake' - 'lib/tasks/meetup.rake' - 'lib/tasks/resource.rake' - 'lib/tasks/users.rake' diff --git a/.ruby-gemset b/.ruby-gemset index 382ffbf3..4dcdb08b 100644 --- a/.ruby-gemset +++ b/.ruby-gemset @@ -1 +1 @@ -operationcode_backend +-global \ No newline at end of file diff --git a/Gemfile b/Gemfile index 14de8d03..4e29918c 100644 --- a/Gemfile +++ b/Gemfile @@ -29,6 +29,7 @@ gem 'rubocop', '~> 0.55.0', require: false gem 'sendgrid-ruby' gem 'sentry-raven' gem 'sidekiq' +gem 'sidekiq-cron' gem 'skylight' gem 'pry-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 30e19a6e..54670d01 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -90,6 +90,8 @@ GEM responders warden (~> 1.2.3) erubis (2.7.0) + et-orbi (1.1.6) + tzinfo execjs (2.7.0) factory_girl (4.8.0) activesupport (>= 3.0.0) @@ -104,6 +106,9 @@ GEM formtastic (3.1.5) actionpack (>= 3.2.13) formtastic_i18n (0.6.0) + fugit (1.1.6) + et-orbi (~> 1.1, >= 1.1.6) + raabro (~> 1.1) geocoder (1.4.3) gibbon (3.0.1) faraday (>= 0.9.1) @@ -185,6 +190,7 @@ GEM pry (>= 0.10.4) public_suffix (3.0.0) puma (3.8.2) + raabro (1.1.6) rack (2.0.5) rack-cors (0.4.1) rack-protection (2.0.3) @@ -251,6 +257,9 @@ GEM connection_pool (~> 2.2, >= 2.2.2) rack-protection (>= 1.5.0) redis (>= 3.3.5, < 5) + sidekiq-cron (1.0.4) + fugit (~> 1.1) + sidekiq (>= 4.2.1) skylight (1.5.0) activesupport (>= 3.0.0) spring (2.0.1) @@ -313,6 +322,7 @@ DEPENDENCIES sendgrid-ruby sentry-raven sidekiq + sidekiq-cron skylight spring spring-watcher-listen (~> 2.0.0) @@ -320,4 +330,4 @@ DEPENDENCIES webmock (~> 3.0, >= 3.0.1) BUNDLED WITH - 1.16.4 + 1.16.5 diff --git a/app/jobs/github_collect_statistics_job.rb b/app/jobs/github_collect_statistics_job.rb new file mode 100644 index 00000000..aa6fb4fd --- /dev/null +++ b/app/jobs/github_collect_statistics_job.rb @@ -0,0 +1,11 @@ +class GithubCollectStatisticsJob + include Sidekiq::Worker + + def perform + limit = GitHub::Client.new.rate_limit + logger.info("Github Ratelimit: #{limit}") + + GitHub::PullRequests.new.fetch_and_save! + GitHub::Issues.new.fetch_and_save! + end +end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 937538b0..95605c50 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,4 +1,7 @@ require 'sidekiq' +require 'sidekiq-cron' + +schedule_file = 'config/job_schedule.yml' Sidekiq.configure_server do |config| # https://stackoverflow.com/questions/28412913/disable-automatic-retry-with-activejob-used-with-sidekiq @@ -10,4 +13,6 @@ } Rails.logger = Sidekiq::Logging.logger ActiveRecord::Base.logger = Sidekiq::Logging.logger + + Sidekiq::Cron::Job.load_from_hash YAML.load_file(schedule_file) if File.exist?(schedule_file) end diff --git a/config/job_schedule.yml b/config/job_schedule.yml new file mode 100644 index 00000000..95fc7441 --- /dev/null +++ b/config/job_schedule.yml @@ -0,0 +1,28 @@ +## Examples of job configurations + +#my_first_job: +# cron: "*/5 * * * *" +# class: "HardWorker" +# queue: hard_worker +# +#second_job: +# cron: "*/30 * * * *" # execute at every 30 minutes +# class: "HardWorker" +# queue: hard_worker_long +# args: +# hard: "stuff" + + +# At 04:00 every day run the meetup member sync job +meetup_member_sync_job: + #cron: "0 4 * * *" + cron: "0 0 5 31 2 ?" # only run Feb 31 + class: "MeetupMemberSyncJob" + queue: "default" + +# At 03:00 every day run the github collect statistics job +github_collect_statistics_job: + #cron: "0 3 * * *" + cron: "0 0 5 31 2 ?" # only run Feb 31 + class: "GithubCollectStatisticsJob" + queue: "default" diff --git a/config/routes.rb b/config/routes.rb index e410d4c7..2a3e8193 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,7 +4,8 @@ ActiveAdmin.routes(self) require 'sidekiq/web' - authenticate :admin_user, ->(u) { u.role.super_admin? } do + require 'sidekiq/cron/web' + authenticate :admin_user, ->(u) { u.role.admin_accessible? } do mount Sidekiq::Web => '/admin/sidekiq' end diff --git a/lib/tasks/git_hub.rake b/lib/tasks/git_hub.rake deleted file mode 100644 index 48eedd3a..00000000 --- a/lib/tasks/git_hub.rake +++ /dev/null @@ -1,15 +0,0 @@ -namespace :git_hub do - desc 'Saves pull request, commit, issue and user details for all repositories to date, to GitHubStatistic and GitHubUser' - task collect_statistics: :environment do - begin - limit = GitHub::Client.new.rate_limit - p 'Rate limit:' - p limit - - GitHub::PullRequests.new.fetch_and_save! - GitHub::Issues.new.fetch_and_save! - rescue => e - Rails.logger.error "When fetching and saving GitHub statistics, experienced this error: #{e}" - end - end -end