[RELEASE] v2.4.1 모니터링 기능 관련 수정

.github/workflows/cd.yml

@@ -171,6 +171,28 @@ jobs:
           target: /home/${{ secrets.USERNAME }}/eatssu-deploy/dev
           rm: true
 
+      - name: prod Grafana Alloy 설정 전송
+        uses: appleboy/scp-action@master
+        if: github.ref_name == 'main'
+        with:
+          host: ${{ secrets.HOST_PROD }}
+          username: ${{ secrets.USERNAME }}
+          key: ${{ secrets.PROD_PRIVATE_KEY }}
+          port: 22
+          source: "monitoring/alloy-config.alloy"
+          target: /home/${{ secrets.USERNAME }}/eatssu-deploy/prod
+
+      - name: dev Grafana Alloy 설정 전송
+        uses: appleboy/scp-action@master
+        if: github.ref_name == 'develop'
+        with:
+          host: ${{ secrets.HOST_DEV }}
+          username: ${{ secrets.USERNAME }}
+          key: ${{ secrets.DEV_PRIVATE_KEY }}
+          port: 22
+          source: "monitoring/alloy-config.alloy"
+          target: /home/${{ secrets.USERNAME }}/eatssu-deploy/dev
+
       - name: prod에 배포
         uses: appleboy/ssh-action@master
         id: deploy-prod
@@ -210,7 +232,9 @@ jobs:
             PUBLISHED_CONTAINERS=$(sudo docker ps -q --filter publish=9000)
             if [ -n "$PUBLISHED_CONTAINERS" ]; then sudo docker rm -f $PUBLISHED_CONTAINERS; fi
             sudo docker pull ${{ secrets.DOCKER_REPO }}/eatssu-prod
+            sudo mkdir -p /var/log/eatssu
             sudo docker run -d --name eatssu-prod --restart unless-stopped -p 9000:9000 -p 127.0.0.1:9001:9001 \
+            -v /var/log/eatssu:/var/log/eatssu \
             --log-driver=awslogs \
             --log-opt awslogs-region=ap-northeast-2 \
             --log-opt awslogs-group=prod-ec2-group \
@@ -224,8 +248,17 @@ jobs:
             -e EATSSU_SLACK_TOKEN="${{ secrets.EATSSU_SLACK_TOKEN }}" \
             -e EATSSU_PROD_DOMAIN="${{ secrets.EATSSU_PROD_DOMAIN }}" \
             -e EATSSU_DEEPL_API_KEY="${{ secrets.EATSSU_DEEPL_API_KEY }}" \
+            -e GRAFANA_BASE_URL="${{ secrets.EATSSU_GRAFANA_BASE_URL }}" \
+            -e GRAFANA_LOKI_DATASOURCE_UID="${{ secrets.EATSSU_GRAFANA_LOKI_DATASOURCE_UID }}" \
+            -e SWAGGER_USER="${{ secrets.SWAGGER_PROD_USER }}" \
+            -e SWAGGER_PASSWORD="${{ secrets.SWAGGER_PROD_PASSWORD }}" \
             ${{ secrets.DOCKER_REPO }}/eatssu-prod
             sudo docker image prune -f
+            ALLOY_CONFIG_SRC="$HOME/eatssu-deploy/prod/monitoring/alloy-config.alloy"
+            sed -e 's|__GRAFANA_TOKEN__|${{ secrets.EATSSU_GRAFANA_TOKEN }}|g' \
+                -e 's|__APP_LABEL__|eatssu-prod|g' \
+                "$ALLOY_CONFIG_SRC" | sudo tee /etc/alloy/config.alloy > /dev/null
+            sudo systemctl restart alloy
 
       - name: dev 서버에 배포
         uses: appleboy/ssh-action@master
@@ -265,7 +298,9 @@ jobs:
             PUBLISHED_CONTAINERS=$(sudo docker ps -q --filter publish=9000)
             if [ -n "$PUBLISHED_CONTAINERS" ]; then sudo docker rm -f $PUBLISHED_CONTAINERS; fi
             sudo docker pull ${{ secrets.DOCKER_REPO }}/eatssu-dev
+            sudo mkdir -p /var/log/eatssu
             sudo docker run -d --name eatssu-dev --restart unless-stopped -p 9000:9000 -p 127.0.0.1:9001:9001 \
+            -v /var/log/eatssu:/var/log/eatssu \
             --log-driver=awslogs \
             --log-opt awslogs-region=ap-northeast-2 \
             --log-opt awslogs-group=dev-ec2-group \
@@ -279,5 +314,12 @@ jobs:
             -e EATSSU_SLACK_TOKEN="${{ secrets.EATSSU_SLACK_TOKEN }}" \
             -e EATSSU_DEV_DOMAIN="${{ secrets.EATSSU_DEV_DOMAIN }}" \
             -e EATSSU_DEEPL_API_KEY="${{ secrets.EATSSU_DEEPL_API_KEY }}" \
+            -e SWAGGER_USER="${{ secrets.SWAGGER_DEV_USER }}" \
+            -e SWAGGER_PASSWORD="${{ secrets.SWAGGER_DEV_PASSWORD }}" \
             ${{ secrets.DOCKER_REPO }}/eatssu-dev
             sudo docker image prune -f
+            ALLOY_CONFIG_SRC="$HOME/eatssu-deploy/dev/monitoring/alloy-config.alloy"
+            sed -e 's|__GRAFANA_TOKEN__|${{ secrets.EATSSU_GRAFANA_TOKEN }}|g' \
+                -e 's|__APP_LABEL__|eatssu-dev|g' \
+                "$ALLOY_CONFIG_SRC" | sudo tee /etc/alloy/config.alloy > /dev/null
+            sudo systemctl restart alloy

monitoring/alloy-config.alloy

@@ -0,0 +1,40 @@
+logging {
+  level = "warn"
+}
+
+prometheus.scrape "spring_app" {
+  targets = [{
+    __address__      = "localhost:9001",
+    __metrics_path__ = "/actuator/prometheus",
+  }]
+  forward_to = [prometheus.remote_write.metrics_hosted_prometheus.receiver]
+}
+
+prometheus.remote_write "metrics_hosted_prometheus" {
+  endpoint {
+    name = "hosted-prometheus"
+    url  = "https://prometheus-prod-49-prod-ap-northeast-0.grafana.net/api/prom/push"
+    basic_auth {
+      username = "3340646"
+      password = "__GRAFANA_TOKEN__"
+    }
+  }
+}
+
+loki.write "grafanacloud" {
+  endpoint {
+    url = "https://logs-prod-030.grafana.net/loki/api/v1/push"
+    basic_auth {
+      username = "1665998"
+      password = "__GRAFANA_TOKEN__"
+    }
+  }
+}
+
+loki.source.file "app_log" {
+  targets = [{
+    __path__    = "/var/log/eatssu/app.log",
+    application = "__APP_LABEL__",
+  }]
+  forward_to = [loki.write.grafanacloud.receiver]
+}

src/main/java/ssu/eatssu/domain/auth/infrastructure/SecurityConfig.java

@@ -1,12 +1,20 @@
 package ssu.eatssu.domain.auth.infrastructure;
 
 import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.core.annotation.Order;
+import org.springframework.security.config.Customizer;
 import org.springframework.security.config.annotation.web.builders.HttpSecurity;
 import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.core.userdetails.User;
+import org.springframework.security.core.userdetails.UserDetails;
+import org.springframework.security.core.userdetails.UserDetailsService;
 import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
 import org.springframework.security.crypto.password.PasswordEncoder;
+import org.springframework.security.provisioning.InMemoryUserDetailsManager;
 import org.springframework.security.web.SecurityFilterChain;
 import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
 import org.springframework.web.cors.CorsConfiguration;
@@ -26,10 +34,14 @@ public class SecurityConfig {
             "/favicon.ico", "/error/**", "/webjars/**", "/h2-console/**"
     };
 
+    private static final String[] SWAGGER_PATHS = {
+            "/swagger-ui.html", "/swagger-ui/**", "/v3/api-docs/**", "/swagger-resources/**"
+    };
+
     private static final String[] AUTH_WHITELIST = {
             "/", "/oauths/kakao", "/oauths/apple", "/menus/**", "/meals/**", "/admin/login", "/oauths/v2/kakao","/oauths/v2/apple",
             "/reviews", "/reviews/menus/**", "/reviews/meals/**", "/v2/reviews/statistics/**",
-            "/v2/reviews/menus/**", "/v2/reviews/meals/**", "/actuator/health", "/actuator/prometheus", "/error-test/**",
+            "/v2/reviews/menus/**", "/v2/reviews/meals/**", "/actuator/health", "/actuator/prometheus",
             "/good-price-stores", "/good-price-stores/**"
     };
 
@@ -47,6 +59,33 @@ public PasswordEncoder passwordEncoder() {
     }
 
     @Bean
+    @Order(1)
+    @Profile({"dev", "prod"})
+    public SecurityFilterChain swaggerFilterChain(HttpSecurity http,
+                                                    UserDetailsService swaggerUserDetailsService) throws Exception {
+        http
+                .securityMatcher(SWAGGER_PATHS)
+                .csrf(csrf -> csrf.disable())
+                .userDetailsService(swaggerUserDetailsService)
+                .authorizeHttpRequests(authorize -> authorize.anyRequest().authenticated())
+                .httpBasic(Customizer.withDefaults());
+        return http.build();
+    }
+
+    @Bean
+    @Profile({"dev", "prod"})
+    public UserDetailsService swaggerUserDetailsService(@Value("${swagger.username}") String username,
+                                                          @Value("${swagger.password}") String password,
+                                                          PasswordEncoder passwordEncoder) {
+        UserDetails swaggerUser = User.withUsername(username)
+                .password(passwordEncoder.encode(password))
+                .roles("SWAGGER")
+                .build();
+        return new InMemoryUserDetailsManager(swaggerUser);
+    }
+
+    @Bean
+    @Order(2)
     protected SecurityFilterChain filterChain(HttpSecurity http) throws Exception {
         http
                 .csrf().disable()

src/main/java/ssu/eatssu/domain/auth/security/JwtAuthenticationFilter.java

@@ -28,7 +28,7 @@ public class JwtAuthenticationFilter extends GenericFilterBean {
     private static final List<String> AUTH_WHITELIST = List.of(
             "/", "/oauths/kakao", "/oauths/apple", "/menus/**", "/meals/**", "/admin/login",
             "/reviews", "/reviews/menus/**", "/reviews/meals/**", "/v2/reviews/statistics/**",
-            "/v2/reviews/menus/**", "/v2/reviews/meals/**", "/actuator/**", "/error-test/**",
+            "/v2/reviews/menus/**", "/v2/reviews/meals/**", "/actuator/**",
             "/swagger-ui.html", "/swagger-ui/**", "/v3/api-docs/**", "/swagger-resources/**",
             "/oauths/valid/token", "/admin/img/**", "/css/**", "/js/**",
             "/favicon.ico", "/error/**", "/webjars/**", "/h2-console/**",

src/main/java/ssu/eatssu/domain/slack/entity/SlackMessageFormat.java

@@ -1,5 +1,7 @@
 package ssu.eatssu.domain.slack.entity;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.MDC;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import ssu.eatssu.domain.inquiry.entity.Inquiry;
@@ -8,15 +10,31 @@
 import ssu.eatssu.domain.user.entity.User;
 import ssu.eatssu.global.handler.response.BaseException;
 
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
 import java.text.MessageFormat;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
 
 @Component
 public class SlackMessageFormat {
 
+    private static final long LOG_LINK_WINDOW_MILLIS = 5 * 60 * 1000;
+
     private static String serverEnv;
+    private static String grafanaBaseUrl;
+    private static String lokiDatasourceUid;
+    private static ObjectMapper objectMapper;
 
-    private SlackMessageFormat(@Value("${server.env:unknown}") String serverEnvValue) {
+    private SlackMessageFormat(@Value("${server.env:unknown}") String serverEnvValue,
+                                @Value("${grafana.base-url:}") String grafanaBaseUrlValue,
+                                @Value("${grafana.loki-datasource-uid:}") String lokiDatasourceUidValue,
+                                ObjectMapper objectMapperValue) {
         SlackMessageFormat.serverEnv = serverEnvValue;
+        SlackMessageFormat.grafanaBaseUrl = grafanaBaseUrlValue;
+        SlackMessageFormat.lokiDatasourceUid = lokiDatasourceUidValue;
+        SlackMessageFormat.objectMapper = objectMapperValue;
     }
 
     public static String sendReport(Report report) {
@@ -113,6 +131,50 @@ public static String sendServerError(Throwable ex, String method, String uri, St
                 userId,
                 args != null && args.length() > 500 ? args.substring(0, 500) + "...(truncated)" : args
         };
-        return messageFormat.format(formatArgs);
+        String message = messageFormat.format(formatArgs);
+
+        String logLink = buildGrafanaLogLink();
+        if (logLink != null) {
+            message += "\n" + logLink;
+        }
+        return message;
+    }
+
+    private static String buildGrafanaLogLink() {
+        String requestId = MDC.get("requestId");
+        if (grafanaBaseUrl == null || grafanaBaseUrl.isBlank()
+                || lokiDatasourceUid == null || lokiDatasourceUid.isBlank()
+                || requestId == null || requestId.isBlank()) {
+            return null;
+        }
+
+        String application = "eatssu-" + serverEnv;
+        String logQl = "{application=\"" + application + "\"} |= \"reqId=" + requestId + "\"";
+        long now = System.currentTimeMillis();
+
+        Map<String, Object> query = new LinkedHashMap<>();
+        query.put("refId", "A");
+        query.put("expr", logQl);
+        query.put("datasource", Map.of("type", "loki", "uid", lokiDatasourceUid));
+        query.put("editorMode", "code");
+
+        Map<String, Object> range = new LinkedHashMap<>();
+        range.put("from", String.valueOf(now - LOG_LINK_WINDOW_MILLIS));
+        range.put("to", String.valueOf(now + LOG_LINK_WINDOW_MILLIS));
+
+        Map<String, Object> pane = new LinkedHashMap<>();
+        pane.put("datasource", lokiDatasourceUid);
+        pane.put("queries", List.of(query));
+        pane.put("range", range);
+        pane.put("compact", false);
+
+        try {
+            String panesJson = objectMapper.writeValueAsString(Map.of("wes", pane));
+            String encodedPanes = URLEncoder.encode(panesJson, StandardCharsets.UTF_8);
+            String url = grafanaBaseUrl + "/explore?schemaVersion=1&panes=" + encodedPanes;
+            return "<" + url + "|Grafana에서 로그 보기>";
+        } catch (Exception e) {
+            return null;
+        }
     }
 }

src/main/java/ssu/eatssu/global/log/ControllerLogAspect.java

@@ -17,6 +17,7 @@
 import ssu.eatssu.domain.auth.security.CustomUserDetails;
 import ssu.eatssu.domain.slack.service.SlackErrorNotifier;
 import ssu.eatssu.global.handler.response.BaseException;
+import ssu.eatssu.global.handler.response.BaseResponseStatus;
 import ssu.eatssu.global.log.annotation.LogMask;
 
 import java.lang.reflect.Field;
@@ -99,7 +100,9 @@ public Object logApi(ProceedingJoinPoint joinPoint) throws Throwable {
             String exceptionType = e.getClass().getSimpleName();
             log.error("EXCEPTION {} {} ({} ms) type={} cause={}", method, uri, time, exceptionType, causeMessage, e);
 
-            slackErrorNotifier.notify(e, method, uri, userId, argsJson);
+            if (shouldNotifySlack(e)) {
+                slackErrorNotifier.notify(e, method, uri, userId, argsJson);
+            }
             throw e;
         }
     }
@@ -124,6 +127,13 @@ private boolean isAuthApi(String uri) {
         return uri != null && uri.startsWith("/oauths/");
     }
 
+    private boolean shouldNotifySlack(Throwable e) {
+        if (e instanceof BaseException baseException) {
+            return BaseResponseStatus.sendSlackNotification(baseException.getStatus());
+        }
+        return true;
+    }
+
     private String getCauseMessage(Throwable e) {
         if (e instanceof BaseException baseException) {
             return baseException.getStatus().getMessage();

src/main/resources/application-dev.yml

@@ -77,6 +77,8 @@ deepl:
 swagger:
   url: ${EATSSU_DEV_DOMAIN}
   description: Test Server Swagger API
+  username: ${SWAGGER_USER}
+  password: ${SWAGGER_PASSWORD}
 
 springdoc:
   swagger-ui:

src/main/resources/application-prod.yml

@@ -78,6 +78,8 @@ deepl:
 swagger:
   url: ${EATSSU_PROD_DOMAIN}
   description: Prod-Server url
+  username: ${SWAGGER_USER}
+  password: ${SWAGGER_PASSWORD}
 
 springdoc:
   swagger-ui:

src/main/resources/logback-spring.xml

@@ -8,6 +8,9 @@
     Thread : 동시성 문제 관련 사용, Request : 요청 단위 -->
     <property name="LOG_PATTERN"
               value="%d{yyyy-MM-dd HH:mm:ss.SSS} %clr([%level]) [%thread] [reqId=%X{requestId}] %cyan(%logger{36}) - %msg%n"/>
+    <!-- 파일 출력용: 색상 코드(%clr) 없이 동일 패턴 -->
+    <property name="FILE_LOG_PATTERN"
+              value="%d{yyyy-MM-dd HH:mm:ss.SSS} [%level] [%thread] [reqId=%X{requestId}] %logger{36} - %msg%n"/>
 
     <!-- 콘솔 출력 (stdout) -->
     <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
@@ -22,8 +25,33 @@
         <appender-ref ref="CONSOLE"/>
     </appender>
 
+    <!-- 파일 출력: Grafana Alloy가 이 파일을 tail해서 Loki로 전송 (호스트 볼륨 마운트 경로와 일치해야 함)
+         /var/log/eatssu는 EC2에만 있는 경로라 prod/dev 프로필에서만 활성화 (test/local에서 켜면 디렉토리가 없어 초기화 실패함) -->
+    <springProfile name="prod,dev">
+        <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+            <file>/var/log/eatssu/app.log</file>
+            <encoder>
+                <pattern>${FILE_LOG_PATTERN}</pattern>
+            </encoder>
+            <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+                <fileNamePattern>/var/log/eatssu/app.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+                <maxFileSize>100MB</maxFileSize>
+                <maxHistory>3</maxHistory>
+                <totalSizeCap>1GB</totalSizeCap>
+            </rollingPolicy>
+        </appender>
+
+        <appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
+            <neverBlock>true</neverBlock>
+            <appender-ref ref="FILE"/>
+        </appender>
+    </springProfile>
+
     <!-- Root Logger -->
     <root level="INFO">
         <appender-ref ref="ASYNC_CONSOLE"/>
+        <springProfile name="prod,dev">
+            <appender-ref ref="ASYNC_FILE"/>
+        </springProfile>
     </root>
 </configuration>