8021961: setAlwaysOnTop doesn't behave correctly in Linux/Solaris under certain scenarios

Reviewed-by: andrew
Backport-of: 09a7c4bc4624dd39ab5500c394c1e298a43a711a

jdk/src/solaris/classes/sun/awt/X11/XWindowPeer.java

@@ -85,7 +85,7 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
     // used for modal blocking to keep existing z-order
     protected XWindowPeer prevTransientFor, nextTransientFor;
     // value of WM_TRANSIENT_FOR hint set on this window
-    private XWindowPeer curRealTransientFor;
+    private XBaseWindow curRealTransientFor;
 
     private boolean grab = false; // Whether to do a grab during showing
 
@@ -1052,13 +1052,23 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
             log.fine("Promoting always-on-top state {0}", Boolean.valueOf(alwaysOnTop));
         }
         XWM.getWM().setLayer(this,
-                             alwaysOnTop ?
+                alwaysOnTop ?
-                             XLayerProtocol.LAYER_ALWAYS_ON_TOP :
+                        XLayerProtocol.LAYER_ALWAYS_ON_TOP :
-                             XLayerProtocol.LAYER_NORMAL);
+                        XLayerProtocol.LAYER_NORMAL);
     }
 
     public void updateAlwaysOnTopState() {
         this.alwaysOnTop = ((Window) this.target).isAlwaysOnTop();
+        if (ownerPeer != null) {
+            XToolkit.awtLock();
+            try {
+                restoreTransientFor(this);
+                applyWindowType();
+            }
+            finally {
+                XToolkit.awtUnlock();
+            }
+        }
         updateAlwaysOnTop();
     }
 
@@ -1102,7 +1112,27 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
         if (!vis && warningWindow != null) {
             warningWindow.setSecurityWarningVisible(false, false);
         }
+        boolean refreshChildsTransientFor = isVisible() != vis;
         super.setVisible(vis);
+        if (refreshChildsTransientFor) {
+            for (Window child : ((Window) target).getOwnedWindows()) {
+                XToolkit.awtLock();
+                try {
+                    if(!child.isLightweight() && child.isVisible()) {
+                        ComponentPeer childPeer = AWTAccessor.
+                                getComponentAccessor().getPeer(child);
+                        if(childPeer instanceof XWindowPeer) {
+                            XWindowPeer windowPeer = (XWindowPeer) childPeer;
+                            restoreTransientFor(windowPeer);
+                            windowPeer.applyWindowType();
+                        }
+                    }
+                }
+                finally {
+                    XToolkit.awtUnlock();
+                }
+            }
+        }
         if (!vis && !isWithdrawn()) {
             // ICCCM, 4.1.4. Changing Window State:
             // "Iconic -> Withdrawn - The client should unmap the window and follow it
@@ -1631,9 +1661,6 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
             window.prevTransientFor = transientForWindow;
             transientForWindow.nextTransientFor = window;
         }
-        if (window.curRealTransientFor == transientForWindow) {
-            return;
-        }
         if (!allStates && (window.getWMState() != transientForWindow.getWMState())) {
             return;
         }
@@ -1645,11 +1672,13 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
             bpw = XlibUtil.getParentWindow(bpw);
         }
         long tpw = transientForWindow.getWindow();
-        while (!XlibUtil.isToplevelWindow(tpw) && !XlibUtil.isXAWTToplevelWindow(tpw)) {
+        XBaseWindow parent = transientForWindow;
+        while (tpw != 0 && ((!XlibUtil.isToplevelWindow(tpw) &&
+                !XlibUtil.isXAWTToplevelWindow(tpw)) || !parent.isVisible())) {
             tpw = XlibUtil.getParentWindow(tpw);
+            parent = XToolkit.windowToXWindow(tpw);
         }
 
-        XBaseWindow parent = transientForWindow;
         if (parent instanceof XLightweightFramePeer) {
             XLightweightFramePeer peer = (XLightweightFramePeer) parent;
             long ownerWindowPtr = peer.getOverriddenWindowHandle();
@@ -1659,7 +1688,7 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
         }
 
         XlibWrapper.XSetTransientFor(XToolkit.getDisplay(), bpw, tpw);
-        window.curRealTransientFor = transientForWindow;
+        window.curRealTransientFor = parent;
     }
 
     /*
@@ -1953,7 +1982,7 @@ class XWindowPeer extends XPanelPeer implements WindowPeer,
         switch (getWindowType())
         {
             case NORMAL:
-                typeAtom = (ownerPeer == null) ?
+                typeAtom = curRealTransientFor == null ?
                                protocol.XA_NET_WM_WINDOW_TYPE_NORMAL :
                                protocol.XA_NET_WM_WINDOW_TYPE_DIALOG;
                 break;

jdk/test/java/awt/Window/MultiWindowApp/ChildAlwaysOnTopTest.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test @summary setAlwaysOnTop doesn't behave correctly in Linux/Solaris under
+ *                certain scenarios
+ * @bug 8021961
+ * @author Semyon Sadetsky
+ * @run main ChildAlwaysOnTopTest
+ */
+
+import javax.swing.*;
+import java.awt.*;
+
+public class ChildAlwaysOnTopTest {
+
+    private static Window win1;
+    private static Window win2;
+    private static Point point;
+
+    public static void main(String[] args) throws Exception {
+        if( Toolkit.getDefaultToolkit().isAlwaysOnTopSupported() ) {
+
+
+            test(null);
+
+            Window f = new Frame();
+            f.setBackground(Color.darkGray);
+            f.setSize(500, 500);
+            try {
+                test(f);
+            } finally {
+                f.dispose();
+            }
+
+            f = new Frame();
+            f.setBackground(Color.darkGray);
+            f.setSize(500, 500);
+            f.setVisible(true);
+            f = new Dialog((Frame)f);
+            try {
+                test(f);
+            } finally {
+                ((Frame)f.getParent()).dispose();
+            }
+        }
+        System.out.println("ok");
+    }
+
+    public static void test(Window parent) throws Exception {
+        SwingUtilities.invokeAndWait(new Runnable() {
+            @Override
+            public void run() {
+                win1 = parent == null ? new JDialog() : new JDialog(parent);
+                win1.setName("top");
+                win2 = parent == null ? new JDialog() : new JDialog(parent);
+                win2.setName("behind");
+                win1.setSize(200, 200);
+                Panel panel = new Panel();
+                panel.setBackground(Color.GREEN);
+                win1.add(panel);
+                panel = new Panel();
+                panel.setBackground(Color.RED);
+                win2.add(panel);
+                win1.setAlwaysOnTop(true);
+                win2.setAlwaysOnTop(false);
+                win1.setVisible(true);
+            }
+        });
+
+        Robot robot = new Robot();
+        robot.delay(200);
+        robot.waitForIdle();
+
+        SwingUtilities.invokeAndWait(new Runnable() {
+            @Override
+            public void run() {
+                point = win1.getLocationOnScreen();
+                win2.setBounds(win1.getBounds());
+                win2.setVisible(true);
+            }
+        });
+
+        robot.delay(200);
+        robot.waitForIdle();
+
+        Color color = robot.getPixelColor(point.x + 100, point.y + 100);
+        if(!color.equals(Color.GREEN)) {
+            win1.dispose();
+            win2.dispose();
+            throw new RuntimeException("alawaysOnTop window is sent back by " +
+                    "another child window setVisible(). " + color);
+        }
+
+        SwingUtilities.invokeAndWait(new Runnable() {
+            @Override
+            public void run() {
+                win2.toFront();
+                if (parent != null) {
+                    parent.setLocation(win1.getLocation());
+                    parent.toFront();
+                }
+            }
+        });
+
+        robot.delay(200);
+        robot.waitForIdle();
+
+        color = robot.getPixelColor(point.x + 100, point.y + 100);
+        if(!color.equals(Color.GREEN)) {
+            win1.dispose();
+            win2.dispose();
+            throw new RuntimeException("alawaysOnTop window is sent back by " +
+                    "another child window toFront(). " + color);
+        }
+
+        SwingUtilities.invokeAndWait(new Runnable() {
+            @Override
+            public void run() {
+                win1.setAlwaysOnTop(false);
+                if (parent != null) {
+                    parent.setVisible(false);
+                    parent.setVisible(true);
+                }
+                win2.toFront();
+            }
+        });
+
+        robot.delay(200);
+        robot.waitForIdle();
+
+        color = robot.getPixelColor(point.x + 100, point.y + 100);
+        if(!color.equals(Color.RED)) {
+            throw new RuntimeException("Failed to unset alawaysOnTop " + color);
+        }
+
+        win1.dispose();
+        win2.dispose();
+    }
+}