Commit 79a3a9ce authored by Bart Ribbers's avatar Bart Ribbers Committed by Leo
Browse files

community/mycroft-core: apply improved XDG patches

Applied patches are all submitted upstream

This removes the need to ship our own start script and makes sure it's
compatible with the patches applied to MSM in the previous commit
parent dc8fbd11
Pipeline #68652 failed with stages
in 57 seconds
From 51820329b69fa37423474aecf8c70d731452dea2 Mon Sep 17 00:00:00 2001
From: Bart Ribbers <bribbers@disroot.org>
Date: Fri, 15 Jan 2021 12:59:25 +0100
Subject: [PATCH 1/2] Replace mycroft-{start,stop} with scripts that can launch
properly installed setups
These scripts will work with system-wide setups
(/usr/bin/mycroft-start), user setups (~/.local/bin/mycroft-start) and
virtual environments (.venv/bin/mycroft-start).
A big benefit is that they don't care where they're installed. They'll
launch from a virtual environment, user setups or system-wide setup, in
that order.
Also make sure these scripts are actually installed by setup.py.
These changes should make it possible to create a PyPi package
installable with pip.
---
bin/mycroft-start | 157 ++++++++++++++++++++++++++++++++++++++++++++--
bin/mycroft-stop | 109 ++++++++++++++++++++++++++++++--
setup.py | 6 +-
3 files changed, 261 insertions(+), 11 deletions(-)
diff --git a/bin/mycroft-start b/bin/mycroft-start
index 65723c2c373..36ddc9fcbe6 100755
--- a/bin/mycroft-start
+++ b/bin/mycroft-start
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
# Copyright 2019 Mycroft AI Inc.
#
@@ -14,8 +14,155 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-SOURCE="${BASH_SOURCE[0]}"
-cd -P "$( dirname "$SOURCE" )"/..
-DIR="$( pwd )"
+script=${0}
+script=${script##*/}
-. "$DIR/start-mycroft.sh" $@
+help() {
+ echo "${script}: Mycroft command/service launcher"
+ echo "usage: ${script} [COMMAND] [restart] [params]"
+ echo
+ echo "Services COMMANDs:"
+ echo " all runs core services: bus, audio, skills, voice"
+ echo " debug runs core services, then starts the CLI"
+ echo " audio the audio playback service"
+ echo " bus the messagebus service"
+ echo " skills the skill service"
+ echo " voice voice capture service"
+ echo " enclosure mark_1 enclosure service"
+ echo
+ echo "Tool COMMANDs:"
+ echo " cli the Command Line Interface"
+ echo
+ echo "Options:"
+ echo " restart (optional) Force the service to restart if running"
+ echo
+ echo "Examples:"
+ echo " ${script} all"
+ echo " ${script} all restart"
+ echo " ${script} bus"
+ echo " ${script} voice"
+
+ exit 1
+}
+
+name_to_script_path() {
+ case ${1} in
+ "bus") _module="mycroft.messagebus.service" ;;
+ "skills") _module="mycroft.skills" ;;
+ "audio") _module="mycroft.audio" ;;
+ "voice") _module="mycroft.client.speech" ;;
+ "cli") _module="mycroft.client.text" ;;
+ "enclosure") _module="mycroft.client.enclosure" ;;
+
+ *)
+ echo "Error: Unknown name '${1}'"
+ exit 1
+ esac
+}
+
+require_process() {
+ name_to_script_path "${1}"
+ if ! pgrep -f "python3 (.*)-m ${_module}" > /dev/null; then
+ launch_background "${1}"
+ fi
+}
+
+launch_process() {
+ name_to_script_path "${1}"
+
+ # Luanch process in foreground
+ echo "Starting $1"
+ python3 -m ${_module} "$_params"
+}
+
+launch_background() {
+ name_to_script_path "${1}"
+
+ if pgrep -f "python3 (.*)-m ${_module}" > /dev/null; then
+ if ($_force_restart); then
+ echo "Restarting: ${1}"
+ mycroft-stop "${1}"
+ else
+ # Already running, no need to restart
+ return
+ fi
+ else
+ echo "Starting background service $1"
+ fi
+
+ # Security warning/reminder for the user
+ if [ "${1}" = "bus" ] ; then
+ echo "CAUTION: The Mycroft bus is an open websocket with no built-in security"
+ echo " measures. You are responsible for protecting the local port"
+ echo " 8181 with a firewall as appropriate."
+ fi
+
+ # Launch process in background
+ # Send logs to XDG Base Directories cache location
+ if [ -n "${XDG_CACHE_HOME+x}" ]; then
+ logdir="$XDG_CACHE_HOME/mycroft"
+ else
+ logdir="$HOME/.cache/mycroft"
+ fi
+
+ if [ ! -d "$logdir" ]; then
+ mkdir -p "$logdir"
+ fi
+
+ python3 -m ${_module} "$_params" >> "$logdir/${1}.log" 2>&1 &
+}
+
+launch_all() {
+ echo "Starting all mycroft-core services"
+ launch_background bus
+ launch_background skills
+ launch_background audio
+ launch_background voice
+ launch_background enclosure
+}
+
+_opt=$1
+_force_restart=false
+shift
+if [ "${1}" = "restart" ] || [ "${_opt}" = "restart" ]; then
+ _force_restart=true
+ if [ "${_opt}" = "restart" ]; then
+ # Support "start-mycroft restart all" as well as "start-mycroft all restart"
+ _opt=$1
+ fi
+ shift
+fi
+_params=$*
+
+case ${_opt} in
+ "all")
+ launch_all
+ ;;
+ "bus")
+ launch_background "${_opt}"
+ ;;
+ "audio")
+ launch_background "${_opt}"
+ ;;
+ "skills")
+ launch_background "${_opt}"
+ ;;
+ "voice")
+ launch_background "${_opt}"
+ ;;
+ "debug")
+ launch_all
+ launch_process cli
+ ;;
+ "cli")
+ require_process bus
+ require_process skills
+ launch_process "${_opt}"
+ ;;
+ "enclosure")
+ launch-background "${_opt}"
+ ;;
+ *)
+ help
+ ;;
+esac
diff --git a/bin/mycroft-stop b/bin/mycroft-stop
index b86b0ea6557..289c736959a 100755
--- a/bin/mycroft-stop
+++ b/bin/mycroft-stop
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
# Copyright 2019 Mycroft AI Inc.
#
@@ -14,8 +14,107 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-SOURCE="${BASH_SOURCE[0]}"
-cd -P "$( dirname "$SOURCE" )"/..
-DIR="$( pwd )"
+script=${0}
+script=${script##*/}
-. "$DIR/stop-mycroft.sh" $@
+help() {
+ echo "${script}: Mycroft service stopper"
+ echo "usage: ${script} [service]"
+ echo
+ echo "Service:"
+ echo " all ends core services: bus, audio, skills, voice"
+ echo " (none) same as \"all\""
+ echo " bus stop the Mycroft messagebus service"
+ echo " audio stop the audio playback service"
+ echo " skills stop the skill service"
+ echo " voice stop voice capture service"
+ echo " enclosure stop enclosure (hardware/gui interface) service"
+ echo
+ echo "Examples:"
+ echo " ${script}"
+ echo " ${script} audio"
+
+ exit 0
+}
+
+process_running() {
+ if [ "$( pgrep -f "python3 (.*)-m mycroft.*${1}" )" ]; then
+ return 0
+ else
+ return 1
+ fi
+}
+
+end_process() {
+ if process_running "$1"; then
+ # Find the process by name, only returning the oldest if it has children
+ pid=$( pgrep -o -f "python3 (.*)-m mycroft.*${1}" )
+ echo "Stopping $1 (${pid})..."
+ kill -SIGINT "${pid}"
+
+ # Wait up to 5 seconds (50 * 0.1) for process to stop
+ c=1
+ while [ $c -le 50 ]; do
+ if process_running "$1"; then
+ sleep 0.1
+ c=$((c + 1))
+ else
+ c=999 # end loop
+ fi
+ done
+
+ if process_running "$1"; then
+ echo "Failed to stop."
+ pid=$( pgrep -o -f "python3 (.*)-m mycroft.*${1}" )
+ echo " Killing $1 (${pid})..."
+ kill -9 "${pid}"
+ echo "Killed."
+ result=120
+ else
+ echo "Stopped."
+ if [ $result -eq 0 ] ; then
+ result=100
+ fi
+ fi
+ fi
+}
+
+result=0 # default, no change
+
+OPT=$1
+shift
+
+case ${OPT} in
+ "all"|"")
+ echo "Stopping all mycroft-core services"
+ end_process "skills"
+ end_process "audio"
+ end_process "speech"
+ end_process "enclosure"
+ end_process "messagebus.service"
+ ;;
+ "bus")
+ end_process "messagebus.service"
+ ;;
+ "audio")
+ end_process "audio"
+ ;;
+ "skills")
+ end_process "skills"
+ ;;
+ "voice")
+ end_process "speech"
+ ;;
+ "enclosure")
+ end_process "enclosure"
+ ;;
+ *)
+ help
+ ;;
+esac
+
+# Exit codes:
+# 0 if nothing changed (e.g. --help or no process was running)
+# 100 at least one process was stopped
+# 120 if any process had to be killed
+exit $result
diff --git a/setup.py b/setup.py
index 963c503e4d2..89eefa496fc 100644
--- a/setup.py
+++ b/setup.py
@@ -80,5 +80,9 @@ def required(requirements_file):
'mycroft-enclosure-client=mycroft.client.enclosure.__main__:main',
'mycroft-cli-client=mycroft.client.text.__main__:main'
]
- }
+ },
+ scripts=[
+ 'bin/mycroft-start',
+ 'bin/mycroft-stop'
+ ]
)
From befd8a1d0a849ff2ccaf2f58e79ffe4d339882c2 Mon Sep 17 00:00:00 2001
From: Bart Ribbers <bribbers@disroot.org>
Date: Fri, 15 Jan 2021 13:18:35 +0100
Subject: [PATCH 2/2] Update README to reflect properly installed setups
---
README.md | 38 +++++++++++++++++++++++++++-----------
1 file changed, 27 insertions(+), 11 deletions(-)
diff --git a/README.md b/README.md
index 1665b0a68dd..9de90003ca7 100644
--- a/README.md
+++ b/README.md
@@ -30,29 +30,45 @@ Mycroft is a hackable open source voice assistant.
# Getting Started
-First, get the code on your system! The simplest method is via git ([git installation instructions](https://gist.github.com/derhuerst/1b15ff4652a867391f03)):
-- `cd ~/`
-- `git clone https://github.com/MycroftAI/mycroft-core.git`
-- `cd mycroft-core`
-- `bash dev_setup.sh`
+Mycroft might be packaged by your distribution already, in that case installing it might be as simple as `apk add mycroft-core` or equivalent for your distribution.
+Otherwise, the simplest method is downloading the latest release on the [Github releases page](https://github.com/MycroftAI/mycroft-core/releases) and unpacking it somewhere.
+Then run `python3 setup.py install` in the unpacked directory.
+For development on Mycroft it's recommended to use Git instead ([git installation instructions](https://gist.github.com/derhuerst/1b15ff4652a867391f03)).
+
+```sh
+cd ~/
+git clone https://github.com/MycroftAI/mycroft-core.git
+cd mycroft-core
+./dev_setup.sh
+```
This script sets up dependencies and a [virtualenv][about-virtualenv]. If running in an environment besides Ubuntu/Debian, Arch or Fedora you may need to manually install packages as instructed by dev_setup.sh.
[about-virtualenv]:https://virtualenv.pypa.io/en/stable/
-NOTE: The default branch for this repository is 'dev', which should be considered a work-in-progress. If you want to clone a more stable version, switch over to the 'master' branch.
+**NOTE:** The default branch for this repository is 'dev', which should be considered a work-in-progress. If you want to clone a more stable version, switch over to the 'master' branch.
# Running Mycroft
-Mycroft provides `start-mycroft.sh` to perform common tasks. This script uses a virtualenv created by `dev_setup.sh`. Assuming you installed mycroft-core in your home directory run:
-- `cd ~/mycroft-core`
-- `./start-mycroft.sh debug`
+Mycroft provides `mycroft-start` to perform common tasks. Assuming you installed mycroft-core via your systems package manager or via `setup.py`:
+
+```sh
+mycroft-start debug
+```
-The "debug" command will start the background services (microphone listener, skill, messagebus, and audio subsystems) as well as bringing up a text-based Command Line Interface (CLI) you can use to interact with Mycroft and see the contents of the various logs. Alternatively you can run `./start-mycroft.sh all` to begin the services without the command line interface. Later you can bring up the CLI using `./start-mycroft.sh cli`.
+The "debug" command will start the background services (microphone listener, skill, messagebus, and audio subsystems) as well as bringing up a text-based Command Line Interface (CLI) you can use to interact with Mycroft and see the contents of the various logs.
+Alternatively you can run `mycroft-start all` to begin the services without the command line interface.
+Later you can bring up the CLI using `mycroft-start cli`.
The background services can be stopped as a group with:
-- `./stop-mycroft.sh`
+
+```sh
+mycroft-stop
+```
+
+If you want to develop for Mycroft, please use `start-mycroft.sh` and `stop-mycroft.sh` instead of the aforementioned commands.
+These provide extra options for development purposes.
# Using Mycroft
From b1da87f6edcdb94c198d52af1ed566c0ccf536a7 Mon Sep 17 00:00:00 2001
From: Bart Ribbers <bribbers@disroot.org>
Date: Fri, 15 Jan 2021 16:29:09 +0100
Subject: [PATCH] Read skills from XDG home directory
Also move over skills from data_dir if they still exist
---
mycroft/configuration/mycroft.conf | 4 +---
mycroft/skills/msm_wrapper.py | 13 +++++--------
mycroft/skills/mycroft_skill/mycroft_skill.py | 2 +-
mycroft/skills/settings.py | 2 +-
mycroft/skills/skill_manager.py | 9 +++++++--
mycroft/skills/skill_updater.py | 4 +++-
pytest.ini | 2 ++
requirements/tests.txt | 1 +
test/unittests/base.py | 3 +++
test/unittests/skills/test_skill_manager.py | 8 +++++---
10 files changed, 29 insertions(+), 19 deletions(-)
diff --git a/mycroft/skills/msm_wrapper.py b/mycroft/skills/msm_wrapper.py
index 4f579f85ac5..27753c678ab 100644
--- a/mycroft/skills/msm_wrapper.py
+++ b/mycroft/skills/msm_wrapper.py
@@ -22,6 +22,7 @@
from collections import namedtuple
from functools import lru_cache
from os import path, makedirs
+from xdg import BaseDirectory
from msm import MycroftSkillsManager, SkillRepo
@@ -33,9 +34,8 @@
[
'platform',
'repo_branch',
- 'repo_cache',
'repo_url',
- 'skills_dir',
+ 'old_skills_dir',
'versioned'
]
)
@@ -70,9 +70,8 @@ def build_msm_config(device_config: dict) -> MsmConfig:
return MsmConfig(
platform=enclosure_config.get('platform', 'default'),
repo_branch=msm_repo_config['branch'],
- repo_cache=path.join(data_dir, msm_repo_config['cache']),
repo_url=msm_repo_config['url'],
- skills_dir=path.join(data_dir, msm_config['directory']),
+ old_skills_dir=path.join(data_dir, msm_config['directory']),
versioned=msm_config['versioned']
)
@@ -94,17 +93,15 @@ def create_msm(msm_config: MsmConfig) -> MycroftSkillsManager:
msm_lock = _init_msm_lock()
LOG.info('Acquiring lock to instantiate MSM')
with msm_lock:
- if not path.exists(msm_config.skills_dir):
- makedirs(msm_config.skills_dir)
+ BaseDirectory.save_data_path('mycroft/skills')
msm_skill_repo = SkillRepo(
- msm_config.repo_cache,
msm_config.repo_url,
msm_config.repo_branch
)
msm_instance = MycroftSkillsManager(
platform=msm_config.platform,
- skills_dir=msm_config.skills_dir,
+ old_skills_dir=msm_config.old_skills_dir,
repo=msm_skill_repo,
versioned=msm_config.versioned
)
diff --git a/mycroft/skills/mycroft_skill/mycroft_skill.py b/mycroft/skills/mycroft_skill/mycroft_skill.py
index c75a8a9b5fa..302c87eae7d 100644
--- a/mycroft/skills/mycroft_skill/mycroft_skill.py
+++ b/mycroft/skills/mycroft_skill/mycroft_skill.py
@@ -124,7 +124,7 @@ def __init__(self, name=None, bus=None, use_settings=True):
# Get directory of skill
#: Member variable containing the absolute path of the skill's root
- #: directory. E.g. /opt/mycroft/skills/my-skill.me/
+ #: directory. E.g. $XDG_DATA_HOME/mycroft/skills/my-skill.me/
self.root_dir = dirname(abspath(sys.modules[self.__module__].__file__))
self.gui = SkillGUI(self)
diff --git a/mycroft/skills/settings.py b/mycroft/skills/settings.py
index c48416afac0..6e70783df0c 100644
--- a/mycroft/skills/settings.py
+++ b/mycroft/skills/settings.py
@@ -99,7 +99,7 @@ def save_settings(skill_dir, skill_settings):
"""Save skill settings to file."""
settings_path = Path(skill_dir).joinpath('settings.json')
- # Either the file already exists in /opt, or we are writing
+ # Either the file already exists or we are writing
# to XDG_CONFIG_DIR and always have the permission to make
# sure the file always exists
if not Path(settings_path).exists():
diff --git a/mycroft/skills/skill_manager.py b/mycroft/skills/skill_manager.py
index b76c555600a..7b1dada8cc8 100644
--- a/mycroft/skills/skill_manager.py
+++ b/mycroft/skills/skill_manager.py
@@ -17,6 +17,8 @@
from glob import glob
from threading import Thread, Event, Lock
from time import sleep, time, monotonic
+import shutil
+from xdg import BaseDirectory
from mycroft.api import is_paired
from mycroft.enclosure.api import EnclosureAPI
@@ -262,7 +264,9 @@ def run(self):
def _remove_git_locks(self):
"""If git gets killed from an abrupt shutdown it leaves lock files."""
- for i in glob(os.path.join(self.msm.skills_dir, '*/.git/index.lock')):
+ for i in glob(os.path.join(
+ BaseDirectory.save_data_path('mycroft/skills'),
+ '*/.git/index.lock')):
LOG.warning('Found and removed git lock file: ' + i)
os.remove(i)
@@ -308,7 +312,8 @@ def _load_skill(self, skill_directory):
return skill_loader if load_status else None
def _get_skill_directories(self):
- skill_glob = glob(os.path.join(self.msm.skills_dir, '*/'))
+ skill_glob = glob(os.path.join(
+ BaseDirectory.save_data_path('mycroft/skills'), '*/'))
skill_directories = []
for skill_dir in skill_glob:
diff --git a/mycroft/skills/skill_updater.py b/mycroft/skills/skill_updater.py
index ace247e2db4..3e7cc9e0b77 100644
--- a/mycroft/skills/skill_updater.py
+++ b/mycroft/skills/skill_updater.py
@@ -53,7 +54,8 @@ def __init__(self, bus=None):
self.config = Configuration.get()
update_interval = self.config['skills']['update_interval']
self.update_interval = int(update_interval) * ONE_HOUR
- self.dot_msm_path = os.path.join(self.msm.skills_dir, '.msm')
+ self.dot_msm_path = os.path.join(
+ BaseDirectory.save_data_path('mycroft/skills'), '.msm')
self.next_download = self._determine_next_download_time()
self._log_next_download_time()
self.installed_skills = set()
diff --git a/pytest.ini b/pytest.ini
index 6634fabcb06..92780db4d40 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,3 +1,5 @@
[pytest]
testpaths = test
norecursedirs = wake_word
+env =
+ XDG_DATA_HOME=/tmp/mycroft-test
diff --git a/requirements/tests.txt b/requirements/tests.txt
index 9e038123e51..3d2e2b9a4c9 100644
--- a/requirements/tests.txt
+++ b/requirements/tests.txt
@@ -2,6 +2,7 @@ coveralls==1.8.2
flake8==3.7.9
pytest==5.2.4
pytest-cov==2.8.1
+pytest-env==0.6.2
cov-core==1.15.0
sphinx==2.2.1
sphinx-rtd-theme==0.4.3
diff --git a/test/unittests/base.py b/test/unittests/base.py
index fed93941769..ee86778fa10 100644
--- a/test/unittests/base.py
+++ b/test/unittests/base.py
@@ -18,6 +18,8 @@
from unittest import TestCase
from unittest.mock import patch
+from xdg import BaseDirectory
+
from .mocks import mock_msm, mock_config, MessageBusMock
@@ -54,3 +56,4 @@ def _mock_log(self):
def tearDown(self):
rmtree(str(self.temp_dir))
+ rmtree(BaseDirectory.save_data_path('mycroft'))
diff --git a/test/unittests/skills/test_skill_manager.py b/test/unittests/skills/test_skill_manager.py
index f2bdde3fa46..8de8168be14 100644
--- a/test/unittests/skills/test_skill_manager.py
+++ b/test/unittests/skills/test_skill_manager.py
@@ -13,6 +13,8 @@
# limitations under the License.
#
from os import path
+from pathlib import Path
+from xdg import BaseDirectory
from unittest import TestCase
from unittest.mock import Mock, patch
@@ -90,7 +92,8 @@ def _mock_skill_updater(self):
self.skill_updater_mock = skill_updater_patch.start()
def _mock_skill_loader_instance(self):
- self.skill_dir = self.temp_dir.joinpath('test_skill')
+ self.skill_dir = (Path(BaseDirectory.save_data_path('mycroft/skills'))
+ .joinpath('test_skill'))
self.skill_loader_mock = Mock(spec=SkillLoader)
self.skill_loader_mock.instance = Mock()
self.skill_loader_mock.instance.default_shutdown = Mock()
@@ -125,8 +128,7 @@ def test_instantiate(self):
)
def test_remove_git_locks(self):
- git_dir = self.temp_dir.joinpath('foo/.git')
- git_dir.mkdir(parents=True)
+ git_dir = Path(BaseDirectory.save_data_path('mycroft/skills/foo/.git'))
git_lock_file_path = str(git_dir.joinpath('index.lock'))
with open(git_lock_file_path, 'w') as git_lock_file:
git_lock_file.write('foo')